/*------------------------------------------------------------------------------*
 * File Name: Import_Utils.c													*
 * Creation: 																	*
 * Purpose: OriginC Source C file												*
 * Copyright (c) ABCD Corp.	2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007		*
 * All Rights Reserved															*
 * 																				*
 * Modification Log:															*
 * SY 08/06/2004 QA70-6780 v8.0116 LLOC_FOR_IMPORT								*
 * SY 08/31/2004 QA70-6472 v8.0127 IMPORT_INFO_STORAGE_CLEANUP					*
 * SY 01/11/2005 QA70-5658 v8.0184 SUPPORT_MULTI_DELIMITER						*
 * EJP 2005-02-16 v8.0193 IMPWIZ_TREAT_CONSECUTIVE_TABS_AND_SPACES_AS_ONE		*
 * EJP 2005-10-20 v8.0321 QA70-7921 XFUNC_TO_IMPORT_FILES						*
 * EJP 2006-10-27 v8.0502 QA70-9073 XUNIT_AND_XLONGNAME_FOR_X_AXIS_TITLE		*
 * Hong 02/26/07 SHOULD_RETURN_FALSE_IF_NO_VARIALBE								*
 * Hong 02/26/07 FIX_SHOULD_NOT_SORT_VARIABLE									*
 * Hong 03/30/07 v8.0593 ROLLBACK_CODE_TO_SORT_VARIABLE_BY_FILENAME_HEADER		*
 *	Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME			*
 *	Hong 09/24/08 v8.0946 FIX_BINARY_IMPORT_FAIL_IF_HAVE_VRIABLE_DEFINED		*
 *	Hong 03/23/09 I_FOUND_INDEX_OUT_OF_RANGE_WHEN_LOAD_FILTER					*
 * EJP 2009-05-29 QA80-13672 FIX_LINE_NUM_IN_VAR_NAME							*
 * EJP 2009-05-29 QA80-13672 DO_NOT_ADD_EMPTY_VALUES							*
 *	Hong 08/10/09 QA80-13711 FIX_REDUNDANT_VARIABLES_EXTRACTED_WHEN_DEALING_EMPTY_LINE
 *------------------------------------------------------------------------------*/

////////////////////////////////////////////////////////////////////////////////////
#include <origin.h>

/// SY 08/06/2004 QA70-6780 v8.0116 LLOC_FOR_IMPORT 
#include "Import_Utils.h"
/// Hong 9/06/06 MOVE_FORM_FILEIMPORT_TO_IMPORT_UTILS
#include "fu_utils.h" 
#include "XFunctionEx.h" 
/// end MOVE_FORM_FILEIMPORT_TO_IMPORT_UTILS

void unpack_data_to_channel(TreeNode& trParent, vector<short>& vData, int nNumChannels)
{
	// Unpacking data to channel if it's multiple-channel
	TreeNode trch, trChData;
	
	if( 1 == nNumChannels )
	{
		trch = tree_check_get_node(trParent, "Ch0");
		ASSERT(trch);
		trChData = tree_check_get_node(trch, IMPTREE_NODE_DATA);
		ASSERT(trChData);

		trChData.SetAttribute(IMPTREE_NODE_DATATYPE, FSI_SHORT);
		trChData.sVals = vData;
	}
	else
	{
		// More than 1 channel
		int nSize = vData.GetSize() / nNumChannels;
	
		vector<short> vs;
		vs.SetSize(nSize);
		
		matrix<short> ms(nSize, nNumChannels);
		ms.SetByVector(vData);
		
		string strChName;
		for( int ich = 0; ich < nNumChannels; ich++ )
		{
			// Get the ich channel's data
			ms.GetColumn(vs, ich);
			
			// Add channel data to tree
			strChName.Format("Ch%d", ich);
			trch = tree_check_get_node(trParent, strChName);
			trChData = tree_check_get_node(trch, IMPTREE_NODE_DATA);
			
			trChData.SetAttribute(IMPTREE_NODE_DATATYPE, FSI_SHORT);
			trChData.sVals = vs;
		}
	}
}
/// end LLOC_FOR_IMPORT


///Danice 1/16/04 v8.0800 MOVE_UTILITY_TO_CPP_FILE
///Jason 12/18/03 v7.5781 ADD_UTILITY_FUNCTION
///Danice 1/17/04 v8.0802 UTILITY_EASY_UNDERSTAND
//void convert_shorts_to_string(string &strTarget, vector<short> &vSource)
void convertShortsToCommaSeparatedStrings(string &strTarget, vector<short> &vSource)
{
	string str;
	StringArray sa;
	for( int n = 0; n < vSource.GetSize(); n++ )
	{
		if( vSource[n] > 0 )
		{
			str.Format("%d", vSource[n]);
			sa.Add(str);
		}
	}
	strTarget.SetTokens(sa, ',');
}

//void convert_string_to_shorts(vector<short> &vTarget, LPCSTR lpcstrSource)
void convertCommaSeparatedStringsToShorts(vector<short>& vTarget, LPCSTR lpcstrSource)
{
	vTarget.SetSize(0);
	
	if( NULL == lpcstrSource )
		return;
	string str(lpcstrSource);

	StringArray sa;
	str.GetTokens(sa, ',');
	
	for( int n = 0; n < sa.GetSize(); n++ )
	{
		sa[n].TrimLeft();
		sa[n].TrimRight();
		if( !sa[n].IsEmpty() )
			vTarget.Add(atoi(sa[n]));
	}
}
///END UTILITY_EASY_UNDERSTAND
///End ADD_UTILITY_FUNCTION
///END MOVE_UTILITY_TO_CPP_FILE

/// EJP 2005-02-16 v8.0193 IMPWIZ_TREAT_CONSECUTIVE_TABS_AND_SPACES_AS_ONE
void delete_consecutive_chars(string &str, char cFind)
{
	char cPrevious = 0;
	int index = 0;
	while( index < str.GetLength() )
	{
		if( cFind == str[index] && cPrevious == str[index] )
			str.Delete(index);
		else
			cPrevious = str[index++];
	}
}
/// end IMPWIZ_TREAT_CONSECUTIVE_TABS_AND_SPACES_AS_ONE

/// SY 01/11/2005 QA70-5658 v8.0184 SUPPORT_MULTI_DELIMITER
// This function will convert all delims in string to first specified delim.
void ConvertMultiDelimToSingle(string &str, vector<int>& vnDelim)
{
	for( int i = 1; i < vnDelim.GetSize(); i++ )
	{
		/// EJP 2005-02-16 v8.0193 IMPWIZ_TREAT_CONSECUTIVE_TABS_AND_SPACES_AS_ONE
		if( vnDelim[i] == ' ' || vnDelim[i] == '\t' )
			delete_consecutive_chars(str, vnDelim[i]);
		/// end IMPWIZ_TREAT_CONSECUTIVE_TABS_AND_SPACES_AS_ONE
		str.Replace(vnDelim[i], vnDelim[0]);
	}
}

/* Current delimiters:      
 *  \t  Tab
 *  \\  Backslash
 */
int CheckDelimiters(string& str)
{
	LPSTR lps, lpd;
	char ch;

	int nNumDelims = str.GetLength();
	LPSTR lpstr = str.GetBuffer(nNumDelims);
	
	lps = lpd = lpstr;
	
	while( (ch = *lpd++ = *lps++) != '\0' ) 
	{
		if( ch == '\\' )
		{
			lpd--;
			switch( ch = *lps++ )
			{
				case 't':
					*lpd = '\t';
					break;
				case '\\':
					*lpd = 92;
					break;
				default:
					*lpd++ = '\\';/* no change if not one of the above */
					*lpd = ch;
					break;
			}
			
			lpd++;
			
			if( ch == '\0' )
				break;
		}
	}
	
	nNumDelims = (lpd - lpstr - 1);
	str.ReleaseBuffer();
	
	//printf("The number of delim = %d\n", nNumDelims);
	return nNumDelims;
}

//--------------------------------------------------------------------------
//	Delimters2String: Translates delimters vector to string
//  9  to '\t'  Tab
//  92  to '\\'  Backslash
//--------------------------------------------------------------------------
BOOL DelimtersToString(vector<int>& vnDelims, string& strDelims)
{
	char cDelim;
	int nDelim;
	string str;
	BOOL bEscapeCode;
	
	for( int i = 0; i < vnDelims.GetSize(); i++ )
	{
		bEscapeCode = TRUE;
		nDelim = vnDelims[i];
		
		if( ( nDelim < 0 ) || ( nDelim > 255 ) )
			return FALSE;
	  
		cDelim = nDelim;
		switch( cDelim )
		{
		case '\t':	   		// '\t' tab
			cDelim = 't';
			break;
	
		case '\\':	   	// '\\' backslash
			//cDelim = '\\';
			break;
	
		default:       // others
			bEscapeCode = FALSE;
		}
	
		if( bEscapeCode )
			strDelims += '\\';
		
		strDelims += cDelim;
	}
	
	return TRUE;
}
/// end SUPPORT_MULTI_DELIMITER

/// EJP 2005-10-20 v8.0321 QA70-7921 XFUNC_TO_IMPORT_FILES
BOOL set_ascimp_delimter(ASCIMP& ai, LPCSTR lpcszDelimiters)
{
	if( !lpcszDelimiters )
		return FALSE;

	int nDelimiters = lstrlen(lpcszDelimiters);
	if( nDelimiters > ASCIMP_MAX_DELIMITERS )
		return FALSE; // too many delimiters, do nothing

	ai.iDelimiter = ASCIMP_DELIM_UNKNOWN;
	ai.cChar = 0;
	memset(ai.cDelimiters, 0, ASCIMP_MAX_DELIMITERS);
	
	if( 1 == nDelimiters )
	{
		switch( *lpcszDelimiters )
		{
		case ',':
			ai.iDelimiter = ASCIMP_DELIM_COMMA;
			break;
		case '\t':
			ai.iDelimiter = ASCIMP_DELIM_TAB;
			break;
		case ' ':
			ai.iDelimiter = ASCIMP_DELIM_SPACE;
			break;
		default:
			ai.iDelimiter = ASCIMP_DELIM_OTHER;
			ai.cChar = *lpcszDelimiters;
			break;
		}
	}
	else
	{
		char* pc = ai.cDelimiters;
		while( *lpcszDelimiters )
			*pc++ = *lpcszDelimiters++;
	}
	return TRUE; // success
}

static BOOL import_asc_files(StringArray& saFileNames, ASCIMP& ai, Worksheet& wks, StringArray& saRanges, int* lpnErrOnFile = NULL, int* lpnErr = NULL)
{
	ASCIMP aiTmp;
	aiTmp = ai;
	string strRange;

	Worksheet wksTarget(wks);
	if( !wksTarget.IsValid() )
		return FALSE;
	Page pgTarget = wksTarget.GetPage();
	if( !pgTarget.IsValid() )
		return FALSE;
	
	saRanges.SetSize(0);
	for( int nFile = 0; nFile < saFileNames.GetSize(); nFile++ )
	{
		if(lpnErrOnFile)
			*lpnErrOnFile = nFile;
		
		if( !get_import_asc_target_wks(wksTarget, wks, aiTmp.iMode, nFile) )
			return FALSE;
		if( 0 == import_asc_file(saFileNames[nFile], ai, wksTarget, strRange, lpnErr) )
			return FALSE;

		saRanges.Add(strRange);
	}
	return TRUE;
}


// This function written for the impASC X-Function.
BOOL import_asc(LPCSTR lpcszFileName, Worksheet& wks, int nDelim, LPCSTR pszPartial, int nMode, string& strRange)
{
	// Create a string array of file names.
	StringArray saFileNames;
	if( '\"' == *lpcszFileName )
	{
		string strFileName = lpcszFileName;
		strFileName.GetTokens(saFileNames);
	}
	else
		saFileNames.Add(lpcszFileName);

	// Get ASCIMP from target wks
	ASCIMP ai;
	wks.GetASCIMP(ai);

	// Check for and update any delimiter over-ride
	if( nDelim >= 0 )
	{
		ai.iDelimited = 1;
		ai.iDelimiter = nDelim;
		if( ASCIMP_DELIM_OTHER == nDelim )
			ai.cChar = ';'; // In this function we use OTHER to indicate semi-colon.
		else
			ai.cChar = 0; // For all other types clear cChar.
	}

	// Check for and update any partial setting over-ride
	if( pszPartial && *pszPartial != '\0')
	{
		/// EJP 2005-10-21
		/// Partial import arg will be supported when Yuri's LabUtil range parser function is ready.
	}

	// Check for and update any import mode over-ride
	if( nMode >= 0 )
		ai.iMode = nMode;

	// Import files
	int nErrFile = -1, nErrCode = -1; //CPY 3/21/06
	StringArray saRanges;
	BOOL bRet = import_asc_files(saFileNames, ai, wks, saRanges, &nErrFile, &nErrCode);
	file_name_array_to_str(strRange, saRanges);

	if(!bRet)
		printf("import_asc error for\n%s\ninternal error code = %d\n",saFileNames[nErrFile], nErrCode);
	
	return bRet;
}

static BOOL get_import_asc_target_wks(Worksheet& wksTarget, Worksheet& wks, int nMode, int nFile)
{
	if( ASCIMP_MODE_REPLACE_DATA == nMode && 0 < nFile )
		nMode = ASCIMP_MODE_NEW_SHEETS;

	switch( nMode )
	{
	case ASCIMP_MODE_REPLACE_DATA:
	case ASCIMP_MODE_APPEND_COLS:
	case ASCIMP_MODE_APPEND_ROWS:
		wksTarget = wks;
		break;
	case ASCIMP_MODE_NEW_BOOKS:
		string strTemplate = LabTalk.System.Wks.DefTemplate$;
		wksTarget.Create(strTemplate);
		break;
	case ASCIMP_MODE_NEW_SHEETS:
		Page pgTarget = wks.GetPage();
		if( !pgTarget.IsValid() )
			return FALSE;
		int nLayer = pgTarget.AddLayer();
		if( nLayer < 0 )
			return FALSE;
		wksTarget = pgTarget.Layers(nLayer);
		break;
	default:
		return FALSE;
	}
	return wksTarget.IsValid();
}


// return UID of data range if success, otherwise return 0
static uint import_asc_file(LPCSTR lpcszFileName, ASCIMP& ai, Worksheet& wks, string& strRange, int* lpnErr = NULL)
{
//	ASCIMP ai;
//	if( AscImpReadFileStruct(lpcszFileName, &ai) )
//		return FALSE; // Failed to init ASCIMP for specified file.

//	ai.iDelimited = 1;
//	ai.iDelimiter = nDelim;
//	if( ASCIMP_DELIM_OTHER == nDelim )
//		ai.cChar = ';'; // In this function we use OTHER to indicate semi-colon.
//	else
//		ai.cChar = 0; // For all other types clear cChar.

//	if( pszPartial && *pszPartial != '\0')
//	{
//		/// EJP 2005-10-21
//		/// Partial import arg will be supported when Yuri's LabUtil range parser function is ready.
//	}
//	ai.iMode = nMode;
	
	ASCIMPRESULT air;
	int nRet = wks.ImportASCII(lpcszFileName, ai, &air);
	if( 0 == nRet )
	{
		DataRange dr;
		dr = (DataRange)Project.GetObject(air.uid);
		if(dr)
			strRange = dr.GetDescription();
		return air.uid;
	}
	if(lpnErr)
		*lpnErr = nRet;
	return 0;
}
/// end XFUNC_TO_IMPORT_FILES

/// Hong 9/06/06 REMOVE_BINIMP
bool ReadBinaryHeaderParam(string &strDest, file &fil, int nType, int nOffset, int nSize, bool bLittleEndian)
{
	if( !fil.IsOpen() )
		return false;
	if( fil.Seek(nOffset, file::begin) != nOffset )
		return false;

	string strFormat;
	
	switch( nType )
	{
	case BIP_TYPE_REAL:
		strFormat = "%g";
		switch( nSize )
		{
		case 4:
			float f;
			fil.ReadFloat(&f, 4, 1, bLittleEndian);
			strDest.Format(strFormat, f);
			break;
		case 8:
			double d;
			fil.ReadFloat(&d, 8, 1, bLittleEndian);
			strDest.Format(strFormat, d);
			break;
		default:
			return false;
		}
		break;

	case BIP_TYPE_INT:
	case BIP_TYPE_UINT:
		strFormat = (BIP_TYPE_INT == nType ? "%d" : "%u");
		switch( nSize )
		{
		case 1:
			char n8;
			fil.Read(&n8, 1);
			strDest.Format(strFormat, n8);
			break;
		case 2:
			short n16;
			fil.ReadInt(&n16, 2, 1, bLittleEndian);
			strDest.Format(strFormat, n16);
			break;
		case 4:
			int n32;
			fil.ReadInt(&n32, 4, 1, bLittleEndian);
			strDest.Format(strFormat, n32);
			break;
		default:
			return false;
		}
		break;

	case BIP_TYPE_STRING:
		LPSTR lpstr = strDest.GetBufferSetLength(nSize + 1); // +1 for null terminator
		if( lpstr )
		{
			fil.Read(lpstr, nSize);
			lpstr[nSize] = 0; // be sure string is terminated
			strDest.ReleaseBuffer();
		}
		break;
		
	default:
		return false;
	}
	
	return true;
}
/// end REMOVE_BINIMP

/// Hong 9/06/06 MOVE_FORM_FILEIMPORT_TO_IMPORT_UTILS
static void get_tokens_ignoring_quotes(StringArray &saTokens, LPCSTR lpcszTokens, int iSeparator)
{
	string strToken, strTokens = lpcszTokens;

	int i = strTokens.Find(iSeparator);
	while( i != -1 )
	{
		saTokens.Add(strTokens.Left(i)); // add everything before the separator into the string array
		strTokens.Delete(0, i + 1); // delete the separator and everything before it
		i = strTokens.Find(iSeparator);
	}
	saTokens.Add(strTokens);
}
static BOOL import_variables_by_scanning(
	StringArray& saVarNames,
	StringArray& saVarValues,
	LPCSTR lpcszDataFile,
	const TreeNode& tnFilter)
{
	StringArray saHdrLines;
	if( !fuGetHdrLines(tnFilter, saHdrLines, lpcszDataFile) )
		return FALSE;
	if( saHdrLines.GetSize() < 1 )
		return TRUE; // no header lines, no variables

	int nFirstLine, nLastLine, nSeparator;
	if( !fuGetHdrVarScan(tnFilter, nFirstLine, nLastLine, nSeparator) )
		return FALSE;

	int n, nLine;
	
	// First loop assumes each line is formatted as: <name> <separator> <value>
	for( nLine = nFirstLine; nLine <= nLastLine; nLine++ )
	{
		/// Hong 03/23/09 I_FOUND_INDEX_OUT_OF_RANGE_WHEN_LOAD_FILTER
		if ( nLine < 0 || nLine >= saHdrLines.GetSize() )
			break;
		/// end I_FOUND_INDEX_OUT_OF_RANGE_WHEN_LOAD_FILTER
		n = saHdrLines[nLine].Find(nSeparator);
		if( n > 0 )
		{
			saVarNames.Add(saHdrLines[nLine].Left(n));
			saVarValues.Add(saHdrLines[nLine].Mid(n + 1));
		}
	}
	
	// Second loop assumes each line is formatted as: <value1> <separator> <value2> <separator> ... <valueN>
	string strName;
	StringArray saTokens;
	for( nLine = nFirstLine; nLine <= nLastLine; nLine++ )
	{
		try
		{
			/// Hong 08/10/09 QA80-13711 FIX_REDUNDANT_VARIABLES_EXTRACTED_WHEN_DEALING_EMPTY_LINE
			//saHdrLines[nLine].GetTokens(saTokens, nSeparator);
			if ( 0 == saHdrLines[nLine].GetTokens(saTokens, nSeparator) )
				saTokens.SetSize(0);
			/// end FIX_REDUNDANT_VARIABLES_EXTRACTED_WHEN_DEALING_EMPTY_LINE
		}
		catch(int nErr)
		{
			saTokens.SetSize(0);
			get_tokens_ignoring_quotes(saTokens, saHdrLines[nLine], nSeparator);
		}

		for( int nToken = 0; nToken < saTokens.GetSize(); nToken++ )
		{
			/// EJP 2009-05-29 QA80-13672 DO_NOT_ADD_EMPTY_VALUES
			saTokens[nToken].TrimLeft();
			saTokens[nToken].TrimRight();
			if( !saTokens[nToken].IsEmpty() )
			{
			/// end DO_NOT_ADD_EMPTY_VALUES

				/// EJP 2009-05-29 QA80-13672 FIX_LINE_NUM_IN_VAR_NAME
				///strName.Format("L%dV%d", nFirstLine + nLine + 1, nToken + 1);
				strName.Format("L%dV%d", nLine + 1, nToken + 1);
				/// end FIX_LINE_NUM_IN_VAR_NAME
			
				saVarNames.Add(strName);
				saVarValues.Add(saTokens[nToken]);

			} /// EJP 2009-05-29 QA80-13672 DO_NOT_ADD_EMPTY_VALUES
		}
	}
	
	return TRUE;
}

/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
//static void check_and_add_header_variable(StringArray &saNames, StringArray &saValues, LPCSTR lpcszName, LPCSTR lpcszValue)
static void check_and_add_header_variable(StringArray &saNames, StringArray &saValues, LPCSTR lpcszName, LPCSTR lpcszValue, StringArray* psaRawNames = NULL)
/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
{
	string strName(lpcszName);
	//strName.TrimLeft();
	//strName.TrimRight();
	/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
	if ( psaRawNames )
	{
		string			strTemp(strName);
		strTemp.TrimLeft();
		strTemp.TrimRight();
		psaRawNames->Add(strTemp);
	}
	/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
	strName.MakeValidCName('_');
	if( !strName.IsEmpty() )
	{
		string strValue(lpcszValue);
		strValue.TrimLeft();
		strValue.TrimRight();

		saNames.Add(strName);
		saValues.Add(strValue);
	}
}

/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
//BOOL import_and_check_variables_by_scanning(
	//StringArray& saVarNames,
	//StringArray& saVarValues,
	//LPCSTR lpcszDataFile,
	//const TreeNode& tnFilter)
BOOL import_and_check_variables_by_scanning(
	StringArray& saVarNames,
	StringArray& saVarValues,
	LPCSTR lpcszDataFile,
	const TreeNode& tnFilter,
	StringArray* psaRawNames/* = NULL*/
	)
/// end /// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
{
	StringArray saTmpVarNames, saTmpVarValues;
	if( import_variables_by_scanning(saTmpVarNames, saTmpVarValues, lpcszDataFile, tnFilter) )
	{
		for( int n = 0; n < saTmpVarNames.GetSize(); n++ )
			/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
			//check_and_add_header_variable(saVarNames, saVarValues, saTmpVarNames[n], saTmpVarValues[n]);
			check_and_add_header_variable(saVarNames, saVarValues, saTmpVarNames[n], saTmpVarValues[n], psaRawNames);
			/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
		return TRUE;
	}
	return FALSE;	
}

static BOOL insert_file_name_lines(StringArray& saLines, LPCSTR lpcszDataFile)
{
	string str;
	str.Format("%s", lpcszDataFile);
	saLines.InsertAt(0, str);
	str.Format("%s", GetFilePath(lpcszDataFile));
	saLines.InsertAt(1, str);
	str.Format("%s", GetFileName(lpcszDataFile));
	saLines.InsertAt(2, str);
	str.Format("%s", GetFileName(lpcszDataFile, TRUE));
	saLines.InsertAt(3, str);
	return TRUE;
}
BOOL get_lines_for_variables(StringArray& saLines, LPCSTR lpcszDataFile, TreeNode& tnFilter)
{
	saLines.SetSize(0); // make empty
	if( FILTER_TYPE_ASCII == fuGetType(tnFilter) )
		fuGetHdrLines(tnFilter, saLines, lpcszDataFile);
	insert_file_name_lines(saLines, lpcszDataFile);
	return TRUE;	
}

BOOL get_tokens_by_delim(StringArray& saTokens, LPCSTR lpcszLine, TreeNode& tnToken)
{
	// Check if tree is valid
	if( !tnToken || (!tnToken.Delimter && !tnToken.Delimiters) )
		return FALSE;
	
	// Get delimiter(s) from tree into local vector
	vector<int> vnDelim;
	if( tnToken.Delimiter )
		vnDelim.Add(tnToken.Delimiter.nVal);
	else
		vnDelim = tnToken.Delimiters.nVals;

	// Copy caller's line into local mutable string
	string strLine;
	strLine = lpcszLine;

	// Change all delimiter chars in string to single delimiter char and
	// if necessary remove all consecutive white space delimter chars
	if( vnDelim.GetSize() > 1 )
		ConvertMultiDelimToSingle(strLine, vnDelim);
	else if( vnDelim[0] == ' ' || vnDelim[0] == '\t' )
		delete_consecutive_chars(strLine, vnDelim[0]);
	
	// Copy tokens into string array
	strLine.GetTokens(saTokens, vnDelim[0]);
	
	return TRUE;
}

static int get_token_line(TreeNode& tnToken)
{
	int nLine = tree_node_get_int(tnToken.Line);
	if( nLine < 0 )
	{
		/// SY 01/05/2004 QA70-5658 v8.0181 IMPROVE_HDRVAR_EXTRACTION
		///if( -3 <= nLine )
		if( nLine + IW_FN_NUM_LINES >= 0 )
		/// end IMPROVE_HDRVAR_EXTRACTION
			nLine = (nLine * -1) - 1;
	}
	else
	{
		/// SY 01/05/2004 QA70-5658 v8.0181 IMPROVE_HDRVAR_EXTRACTION
		///nLine += 3; // 3 file name lines
		nLine += IW_FN_NUM_LINES;
		/// end IMPROVE_HDRVAR_EXTRACTION
	}
	return nLine;
}

static BOOL get_token_line(string& strLine, StringArray& saLines, TreeNode& tnToken)
{
	int nLine = get_token_line(tnToken);
	if( nLine < 0 || saLines.GetSize() <= nLine )
		return FALSE;
	strLine = saLines[nLine];
	return TRUE;
}
static BOOL get_token_by_delim(string& strToken, StringArray& saLines, TreeNode& tnToken)
{
	string strLine;
	if( !get_token_line(strLine, saLines, tnToken) )
		return FALSE;

	/// EJP 2005-02-16 v8.0193 IMPWIZ_TREAT_CONSECUTIVE_TABS_AND_SPACES_AS_ONE
	/// Old code was not worth keeping around.
	StringArray saTokens;
	if( !get_tokens_by_delim(saTokens, strLine, tnToken) )
		return FALSE;
	
	int nToken = tree_node_get_int(tnToken.Token, -1); // -1 = Token index not specified
	if( nToken < 0 || saTokens.GetSize() <= nToken )
		return FALSE;
	
	strToken = saTokens[nToken];
	/// end IMPWIZ_TREAT_CONSECUTIVE_TABS_AND_SPACES_AS_ONE
	return TRUE;
}
static BOOL get_token_by_pos(string& strToken, StringArray& saLines, TreeNode& tnToken)
{
	string strLine;
	if( !get_token_line(strLine, saLines, tnToken) )
		return FALSE;
	
	int nOffset = tree_node_get_int(tnToken.Offset);
	if( nOffset < 0 || strLine.GetLength() <= nOffset )
		return FALSE;
	
	int nLength = tree_node_get_int(tnToken.Length);
	if( nLength < 1 )
		return FALSE;
	
	strToken = strLine.Mid(nOffset, nLength);
	return TRUE;
}
static BOOL get_token(string& strToken, StringArray& saLines, TreeNode& tnToken)
{
	BOOL bRet = FALSE;
	if( tnToken.Constant )
	{
		strToken = tnToken.Constant.strVal;
		bRet = TRUE;
	}
	else if( tnToken.Delimiter || tnToken.Delimiters )
		bRet = get_token_by_delim(strToken, saLines, tnToken);
	else if( tnToken.Offset )
		bRet = get_token_by_pos(strToken, saLines, tnToken);
	if( bRet )
	{
		strToken.TrimLeft();
		strToken.TrimRight();
	}
	return bRet;
}

/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
//BOOL get_variable_from_data_file(string& strName, string& strValue, StringArray& saLines, TreeNode& tnVar)
BOOL get_variable_from_data_file(string& strName, string& strValue, StringArray& saLines, TreeNode& tnVar, string* pstrRawName/* = NULL*/)
/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
{
	if( get_token(strName, saLines, tnVar.Name) && get_token(strValue, saLines, tnVar.Value) )
	{
		/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
		if ( pstrRawName )
		{
			*pstrRawName = strName;
		}
		/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
		strName.MakeValidCName('_');
		return TRUE;
	}
	return FALSE;
}

BOOL CallImportWizardXFunction(Layer& lyTarget, LPCSTR lpcszFile, int nFile, TreeNode& tnFilter, TreeNode& tnInfo, int event, int& nRet)
{
	string strXFuncName;
	if( !fuGetXFunctionName(tnFilter, strXFuncName) )
		return FALSE;

	XFunction xf;
	TreeNode trXF;
	bool bSkipHelpNodes = true;
	if(!xf.Load(&trXF, strXFuncName, 0, true, bSkipHelpNodes, true))
		return false;

	//---- CPY 7/18/06 MATRIXLAYER_SUPPORT_FOR_IMPORT
	//Worksheet wks = lyTarget;///pgTarget.Layers();
	Datasheet wks = lyTarget;
	//----
	DataRange dr;
	/// AW 07/17/2006 IMPORT_MULTI_FILES_IN_ONE_WKS
	//dr.Add(wks,0);
	int nFirstTargetCol = 0;
	if ( ASCIMP_MODE_APPEND_COLS == fuGetImportMode(tnFilter) )
	{
		nFirstTargetCol = wks_find_empty_column(wks);
		if ( nFirstTargetCol < 0 )
		{
			Worksheet wksTemp = wks;       //// AW !!!!!! later need think about Matrixlayer and graphLayer
			nFirstTargetCol = wksTemp.GetNumCols();;
			wksTemp.AddCol();
		}
	}
	if(wks)
		dr.Add(wks,nFirstTargetCol);
	/// END IMPORT_MULTI_FILES_IN_ONE_WKS
	
	//if( !xf.SetArg("fname", lpcszFile) )
	if( !xf.SetArg("fname", (string)lpcszFile) )//Hong 8/17/06 NEED_TRANSFORM_INTO_STRING
		return FALSE;
	if( !xf.SetArg("orng", dr) )
		return FALSE;
	xf.SetArg("trFilter", tnFilter);
	xf.SetArg("findex", nFile);
	xf.SetArg("finfo", tnInfo);
	
	return xf.Evaluate();
}
/// end MOVE_FORM_FILEIMPORT_TO_IMPORT_UTILS
/// Hong 9/26/06 ADD_VARIABLE_TO_PAGE_INFO
static BOOL is_variable_from_file_name(TreeNode& tnVar)
{
	if( tnVar && tnVar.Value.Line && tnVar.Value.Line.nVal < 0 )
		return TRUE;
	return FALSE;
}

/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
//BOOL get_variables_from_data_file(
	//StringArray& saVarNames, StringArray& saVarValues,
	//LPCSTR lpcszDataFile,
	//TreeNode& tnFilter,
	//BOOL bFromFileName)
BOOL get_variables_from_data_file(
	StringArray& saVarNames, StringArray& saVarValues,
	LPCSTR lpcszDataFile,
	TreeNode& tnFilter,
	BOOL bFromFileName,
	StringArray* psaVarNiceNames/* = NULL*/
	)
/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
{
	StringArray saLines;
	insert_file_name_lines(saLines, lpcszDataFile);
	if( !bFromFileName )
		fuGetHdrLines(tnFilter, saLines, lpcszDataFile);
	
	TreeNode tnVars;
	if( !fuGetVars(tnFilter, tnVars) )
		return FALSE;
	
	string strName, strValue;
	string strRawName; /// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
	foreach( TreeNode tn in tnVars.Children )
	{
	/// Hong 03/30/07 v8.0593 ROLLBACK_CODE_TO_SORT_VARIABLE_BY_FILENAME_HEADER
	// required by Easwar
	///// Hong 02/26/07 FIX_SHOULD_NOT_SORT_VARIABLE
	//// all variable will be extract in one function, no need to do sort, keep user's order is better
		if( (bFromFileName && is_variable_from_file_name(tn)) || (!bFromFileName && !is_variable_from_file_name(tn)) )
		{
			/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
			//if( get_variable_from_data_file(strName, strValue, saLines, tn) )
			if ( get_variable_from_data_file(strName, strValue, saLines, tn, &strRawName) )
			/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
			{
				saVarNames.Add(strName);
				saVarValues.Add(strValue);
				/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
				if ( psaVarNiceNames )
					psaVarNiceNames->Add(strRawName);
				/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
			}
		}	
	/// end FIX_SHOULD_NOT_SORT_VARIABLE
	/// end ROLLBACK_CODE_TO_SORT_VARIABLE_BY_FILENAME_HEADER
	}
	/// Hong 02/26/07 SHOULD_RETURN_FALSE_IF_NO_VARIALBE
	if(saVarNames.GetSize() < 1)
		return false;
	/// end FIX_FUNC_ALWAYS_RETURN_TRUE
	return TRUE;
}
BOOL import_variables_by_function(
	StringArray& saVarNames,
	StringArray& saVarValues,
	LPCSTR lpcszDataFile,
	const TreeNode& tnFilter)
{
	/// Hong 02/07/07 ADD_VARIALBE_BY_FUNC_SUPPORT
	//return FALSE;
	StringArray saHdrLines;
	if( !fuGetHdrLines(tnFilter, saHdrLines, lpcszDataFile) )
		return FALSE;
	
	PFNHDRVARFUNC pfn = fuGetExtractVarOCFuncPtr(tnFilter);		
	if(pfn)
		pfn(saVarNames, saVarValues, saHdrLines, tnFilter);
	else
		return FALSE;
	
	return TRUE;
	///  end ADD_VARIALBE_BY_FUNC_SUPPORT
}

/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
//BOOL import_variables_from_ascii_file(
	//StringArray& saVarNames,
	//StringArray& saVarValues,
	//LPCSTR lpcszDataFile,
	//const TreeNode& tnFilter)
BOOL import_variables_from_ascii_file(
	StringArray& saVarNames,
	StringArray& saVarValues,
	LPCSTR lpcszDataFile,
	const TreeNode& tnFilter,
	StringArray* psaVarNiceNames/* = NULL*/
	)
/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
{
	int n;
	StringArray saTmpVarNames, saTmpVarValues;

	if( fuIsExtractVarByDelimiter(tnFilter) || fuIsExtractVarByPosition(tnFilter) )
	{
		/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
		/*
		get_variables_from_data_file(saTmpVarNames, saTmpVarValues, lpcszDataFile, tnFilter, FALSE); // FALSE == from header lines
		for( n = 0; n < saTmpVarNames.GetSize(); n++ )
			check_and_add_header_variable(saVarNames, saVarValues, saTmpVarNames[n], saTmpVarValues[n]);
		*/
		get_variables_from_data_file(saVarNames, saVarValues, lpcszDataFile, tnFilter, FALSE, psaVarNiceNames); // FALSE == from header lines
		/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
	}

	if( fuIsExtractVarByScanning(tnFilter) )
	{
		///import_variables_by_scanning(saTmpVarNames, saTmpVarValues, lpcszDataFile, tnFilter);
		///for( n = 0; n < saTmpVarNames.GetSize(); n++ )
		///	check_and_add_header_variable(saVarNames, saVarValues, saTmpVarNames[n], saTmpVarValues[n]);
		/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
		//import_and_check_variables_by_scanning(saVarNames, saVarValues, lpcszDataFile, tnFilter);
		import_and_check_variables_by_scanning(saVarNames, saVarValues, lpcszDataFile, tnFilter, psaVarNiceNames);
		/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
	}
	
	if( fuIsExtractVarByOCFunc(tnFilter) )
	{				
		import_variables_by_function(saTmpVarNames, saTmpVarValues, lpcszDataFile, tnFilter);		
		for( n = 0; n < saTmpVarNames.GetSize(); n++ )
			/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
			//check_and_add_header_variable(saVarNames, saVarValues, saTmpVarNames[n], saTmpVarValues[n]);
			check_and_add_header_variable(saVarNames, saVarValues, saTmpVarNames[n], saTmpVarValues[n], psaVarNiceNames);
			/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
	}
	/// Hong 02/26/07 SHOULD_RETURN_FALSE_IF_NO_VARIALBE
	if(saVarNames.GetSize() < 1)
		return false;
	/// end FIX_FUNC_ALWAYS_RETURN_TRUE
	return TRUE;
}

/// Hong 08/11/08 QA80-11980 VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
//BOOL import_binary_header_variables(StringArray& saVarNames, StringArray& saVarValues, LPCSTR lpcszDataFile, const TreeNode& tnFilter)
BOOL import_binary_header_variables(StringArray& saVarNames, StringArray& saVarValues, LPCSTR lpcszDataFile, const TreeNode& tnFilter, StringArray* psaVarNiceNames/* = NULL*/)
/// end VARAIBLE_TO_USER_PARAMETER_USING_NICE_NAME
{
	BINIMP binimp;
	if( !fuGetBINIMP(tnFilter, binimp) )
		return false;
		
	if( !fuGetHeaderParamNames(tnFilter, saVarNames) )
		return false;
	
	/// Hong 09/24/08 v8.0946 FIX_BINARY_IMPORT_FAIL_IF_HAVE_VRIABLE_DEFINED
	if ( psaVarNiceNames )
		*psaVarNiceNames = saVarNames;
	/// end FIX_BINARY_IMPORT_FAIL_IF_HAVE_VRIABLE_DEFINED

	file fil;
	if( !fil.Open(lpcszDataFile, file::modeRead|file::typeBinary) )
		return false;

	string str;
	int nType, nOffset, nSize;
	for( int n = 0; n < saVarNames.GetSize(); n++ )
	{
		str.Empty();
		
		if( fuGetBinHeaderParam(tnFilter, saVarNames[n], nType, nOffset, nSize) )
			ReadBinaryHeaderParam(str, fil, nType, nOffset, nSize, !binimp.iBigEndian);
		
		saVarValues.Add(str);
	}

	fil.Close();
	/// Hong 02/26/07 SHOULD_RETURN_FALSE_IF_NO_VARIALBE
	if(saVarNames.GetSize() < 1)
		return false;
	/// end FIX_FUNC_ALWAYS_RETURN_TRUE
	return true;
}

/// END ADD_VARIABLE_TO_PAGE_INFO

/// EJP 2006-10-27 v8.0502 QA70-9073 XUNIT_AND_XLONGNAME_FOR_X_AXIS_TITLE
#define SZ_SYSTEM_DISPLAY_SECTION "system.display"
bool setColXUnitsAndLongName(Column& col, LPCSTR lpcszXUnits, LPCSTR lpcszXLongName)
{
	if( col )
	{
		Tree tr;
		if( info_get_section(col, tr, SZ_SYSTEM_DISPLAY_SECTION) )
		{
			if( lpcszXUnits )
				tr.XUnits.strVal = lpcszXUnits;
			if( lpcszXLongName )
				tr.XLongName.strVal = lpcszXLongName;
			return info_set_section(col, tr, SZ_SYSTEM_DISPLAY_SECTION);
		}
	}
	return false;
}
/// end XUNIT_AND_XLONGNAME_FOR_X_AXIS_TITLE

///---Sim 01-30-2007 GENERAL_IMPORT_MODE_LIST
//string getImportModeListString(string &strMapping, DWORD dwCtrl)
	//// = NULL,
	//// 	IW_FLAG_IMPMODE_AUTO | IW_FLAG_IMPMODE_REPLACE | IW_FLAG_IMPMODE_NEW_BOOKS
	////	 | IW_FLAG_IMPMODE_NEW_SHEETS | IW_FLAG_IMPMODE_NEW_COLS | IW_FLAG_IMPMODE_NEW_ROWS)
//{
	//StringArray saList;
	//vector<int> vTempMapping;
	//
	//if ( dwCtrl & IW_FLAG_IMPMODE_AUTO)
	//{
		//saList.Add("Auto");
		//vTempMapping.Add(ASCIMP_MODE_AUTO);
	//}
		//
	//if ( dwCtrl & IW_FLAG_IMPMODE_REPLACE)
	//{
		//saList.Add("Replace Exisiting Data");
		//vTempMapping.Add(ASCIMP_MODE_REPLACE_DATA);
	//}
	//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_BOOKS)
	//{
		//saList.Add("Start New Books");
		//vTempMapping.Add(ASCIMP_MODE_NEW_BOOKS);
	//}
//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_SHEETS)
	//{
		//saList.Add("Start New Sheets");
		//vTempMapping.Add(ASCIMP_MODE_NEW_SHEETS);
	//}
//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_COLS)
	//{
		//saList.Add("Start New Columns");
		//vTempMapping.Add(ASCIMP_MODE_APPEND_COLS);
	//}
//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_ROWS)
	//{
		//saList.Add("Start New Rows");
		//vTempMapping.Add(ASCIMP_MODE_APPEND_ROWS);
	//}
//
	//if ( NULL != strMapping )
	//{
		//StringArray saMapping;
		//for ( int ii = 0; ii < vTempMapping.GetSize(); ii++ )
			//saMapping.Add(vTempMapping[ii]);
		////saMapping = vTempMapping;
		//strMapping.SetTokens(saMapping, '|');
	//}
	//
	//string strList;
	//strList.SetTokens(saList, '|');
//
	//return strList;
//}
//int getImportModeFromListIndex(int nListIndex, DWORD dwCtrl)
	//// = IW_FLAG_IMPMODE_AUTO | IW_FLAG_IMPMODE_REPLACE | IW_FLAG_IMPMODE_NEW_BOOKS
	////	 | IW_FLAG_IMPMODE_NEW_SHEETS | IW_FLAG_IMPMODE_NEW_COLS | IW_FLAG_IMPMODE_NEW_ROWS)
//{
	///* following enum copied from import_utils.h
	//IW_LIST_IMPMODE_AUTO = 0,
	//IW_LIST_IMPMODE_REPLACE,
	//IW_LIST_IMPMODE_NEW_BOOKS,
	//IW_LIST_IMPMODE_NEW_SHEETS,
	//IW_LIST_IMPMODE_NEW_COLS,
	//IW_LIST_IMPMODE_NEW_ROWS
	//*/
	//
	//// construct list of import mode as custom setting
	//vector<int> vImportMode;
	//
	//if ( dwCtrl & IW_FLAG_IMPMODE_AUTO)
		//vImportMode.Add(ASCIMP_MODE_AUTO);
	//
	//if ( dwCtrl & IW_FLAG_IMPMODE_REPLACE)
		//vImportMode.Add(ASCIMP_MODE_REPLACE_DATA);
	//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_BOOKS)
		//vImportMode.Add(ASCIMP_MODE_NEW_BOOKS);
//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_SHEETS)
		//vImportMode.Add(ASCIMP_MODE_NEW_SHEETS);
//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_COLS)
		//vImportMode.Add(ASCIMP_MODE_APPEND_COLS);
//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_ROWS)
		//vImportMode.Add(ASCIMP_MODE_APPEND_ROWS);
	//
	//// get import mode
	//if( 0 <= nListIndex && nListIndex < vImportMode.GetSize() )
		//return vImportMode[nListIndex];
	//return ASCIMP_MODE_AUTO;
//}
//int getListIndexFromImportMode(int nImportMode, DWORD dwCtrl)
	//// = IW_FLAG_IMPMODE_AUTO | IW_FLAG_IMPMODE_REPLACE | IW_FLAG_IMPMODE_NEW_BOOKS
	////	 | IW_FLAG_IMPMODE_NEW_SHEETS | IW_FLAG_IMPMODE_NEW_COLS | IW_FLAG_IMPMODE_NEW_ROWS)
//{
	///* following defines copied from OC_Types.h
	//#define ASCIMP_MODE_REPLACE_DATA		0
	//#define ASCIMP_MODE_APPEND_COLS			1
	//#define ASCIMP_MODE_APPEND_ROWS			2
	//#define ASCIMP_MODE_NEW_BOOKS			3
	//#define ASCIMP_MODE_NEW_SHEETS			4
	//#define ASCIMP_MODE_AUTO				5
	//*/
	//
	//// construct list of import mode as custom setting
	//vector<int> vListIndex = {
		//0,//ASCIMP_MODE_REPLACE_DATA, Default is first item of list
		//0,//ASCIMP_MODE_APPEND_COLS, Default is first item of list
		//0,//ASCIMP_MODE_APPEND_ROWS, Default is first item of list
		//0,//ASCIMP_MODE_NEW_BOOKS, Default is first item of list
		//0,//ASCIMP_MODE_NEW_SHEETS, Default is first item of list
		//0 //ASCIMP_MODE_AUTO, Default is first item of list
	//};
	//
	//int nNumList = 0;
	//if ( dwCtrl & IW_FLAG_IMPMODE_AUTO)
		//vListIndex[ASCIMP_MODE_AUTO] = nNumList++;
	//
	//if ( dwCtrl & IW_FLAG_IMPMODE_REPLACE)
		//vListIndex[ASCIMP_MODE_REPLACE_DATA] = nNumList++;
	//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_BOOKS)
		//vListIndex[ASCIMP_MODE_NEW_BOOKS] = nNumList++;
//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_SHEETS)
		//vListIndex[ASCIMP_MODE_NEW_SHEETS] = nNumList++;
//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_COLS)
		//vListIndex[ASCIMP_MODE_APPEND_COLS] = nNumList++;
//
	//if ( dwCtrl & IW_FLAG_IMPMODE_NEW_ROWS)
		//vListIndex[ASCIMP_MODE_APPEND_ROWS] = nNumList++;
	//
	//
	//if( IS_IMPORT_MODE(nImportMode) )
		//return vListIndex[nImportMode];
	//return 0;//Default is first item of list
//}
///---END GENERAL_IMPORT_MODE_LIST
