/*------------------------------------------------------------------------------*
 * File Name:	DiademFile.cpp 													*
 * Creation: 	Cheney 2006-12-24												*
 * Purpose: OriginC Source C file												*
 * Copyright (c) ABCD Corp.	2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010		*
 * All Rights Reserved															*
 * 																				*
 * Modification Log:															*
 *	Cheney 2007-1-8 SUPPORT_UNICODE_FILE										*
 *	Cheney 2007-1-9 HANDLE_MONOTONE_OF_IMPLICIT_DATA							*
 *	Cheney 2007-3-7 DATASET_NOW_ONLY_SUPPORT_DOUBLE_TYPE_COLUMN					*
 *	Sim 04-18-2007 ADD_RANGE_INFO												*
 *	Hong 04/27/07 QA80-9247-P18 FIX_FAIL_WHEN_MULTI_FILE_WITH_START_NEW_COLUMN	*
 *	Sim 05-28-2007 QA80-9828-P3 REPEAR_IMPORT_NEED_ALLOW_PASS_DATA_SELECTION	*
 *	Hong 07/11/07 QA9828-P4 v8.0657 FIX_IMPORT_AS_BINARY_REIMPORT_UNWANTED_MISSING_VALUE
 *	Hong 07/20/07 QA80-9828-P7 v8.0664 FIX_REIMPORT_BINARY_DIADEM_DUPLICATE_STORAGE_INFO
 *	Hong 12/07/07 v8.0763 FIX_DIADEM_BINARY_FAIL_IMPORT_CAUSED_BY_MARCO_DUPLICATE
 *	Hong 07/17/08 v8.0903 FIX_SOME_IMPORT_HAVE_DUPLICATE_SHEET_NAME_NO_AUTO_ENUM*
 *	Hong 04/13/09 FIX_DUPLICATE_SNAME_AFTER_RENAME_LEAD_TO_COLUMN_MISSING		*
 *	Hong 04/14/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED	*
 *	Sim 02-04-2010 QA81-15063 MOVE_IMP_FILE_INFO_OUT_FROM_COL_USER_INFO_TREE	*
 *	Sim 02-05-2010 QA81-15063 ROLL_BACK_MOVE_COL_INFO_OUT_OF_USER_TREE			*
 *	Hong 08/04/10 ORG-635-P1 FIX_BLOCK_DATA_FAIL_IMPORT_WHEN_STORED_IN_MULTI_FILES
 *	Hong 08/04/10 ORG-635-P2 FIX_FAIL_IMPORT_CORRECT_CHANNEL_WHEN_SKIP_BLOCK_DATA
 *------------------------------------------------------------------------------*/
 
////////////////////////////////////////////////////////////////////////////////////
// Including the system header file Origin.h should be sufficient for most Origin
// applications and is recommended. Origin.h includes many of the most common system
// header files and is automatically pre-compiled when Origin runs the first time.
// Programs including Origin.h subsequently compile much more quickly as long as
// the size and number of other included header files is minimized. All NAG header
// files are now included in Origin.h and no longer need be separately included.
//
// Right-click on the line below and select 'Open "Origin.h"' to open the Origin.h
// system header file.
#include <Origin.h>
////////////////////////////////////////////////////////////////////////////////////


////////////////////////////////////////////////////////////////////////////////////
// Include your own header files here.
#include <oExtFile.h>
#include "DiademFile.h"
#include "fu_utils.h"
#include "AscImpOptions.h"

////////////////////////////////////////////////////////////////////////////////////
// Start your functions here.

#define	STR_DIADEM_VALID_FLAG 			"DIAEXTENDED  {@:ENGLISH"
#define	STR_DIADEM_GENERAL_HEADER_BEGIN "#BEGINGLOBALHEADER"
#define	STR_DIADEM_GENERAL_HEADER_END	"#ENDGLOBALHEADER"
#define	STR_DIADEM_CHANNEL_HEADER_BEGIN "#BEGINCHANNELHEADER"
#define	STR_DIADEM_CHANNEL_HEADER_END	"#ENDCHANNELHEADER"

#define STR_HEADER_TAG_NAME				"Header"
#define STR_CHANNELS_TAG_NAME			"Channels"
#define STR_CHANNEL_TAG_NAME			"Channel"

#define STR_DATA_STORE_IMPLICIT			"IMPLICIT"
#define STR_DATA_STORE_EXPLICIT			"EXPLICIT"

#define STR_NO_VALUE					"NOVALUE"

#define STR_MONOTONE_FLAG				"MonotoneFlag"

#define STR_NO							"No"
#define STR_YES							"Yes"
#define STR_EMPTY						""

#define STR_DATA_TYPE_REAL_64			"REAL64"

#define STR_CRLF						"CRLF"

#define STR_COMMA						","

#define STR_TIME						"Time"

#define DAY_OF_A_YEAR					86400
#define DIFF_TIME_OFFSET_BETTEEN_TDM_OC 1721060

#define DEFAULT_FILE_MISSING			9.9e34

#define CHAR_COMMA						','
#define CHAR_TAB						'	'
#define CHAR_SPACE						' '
#define CHAR_SEMINAL					';'


#define DELIMETERS_ARRAY_SIZE			5

//the index in chDelimiter
#define SEP_TAB_INDEX					0
#define SEP_COMMA_INDEX					1
#define SEP_SPACE_INDEX					2
#define SEP_SEMINAL_INDEX				3
#define SEP_OTHER_INDEX					4

//the value for iDelimiter
/// Hong 12/07/07 v8.0763 FIX_DIADEM_BINARY_FAIL_IMPORT_CAUSED_BY_MARCO_DUPLICATE
//#define DMT_TAB_VAL						2
//#define DMT_COMMA_VAL					1
//#define DMT_SPACE_VAL					3
//#define DMT_SEMINAL_OR_OTHER_VAL		4
/// end FIX_DIADEM_BINARY_FAIL_IMPORT_CAUSED_BY_MARCO_DUPLICATE

#define TRIM_STRING_LEFT_RIGHT_SPACE(_str) _str.TrimLeft(); _str.TrimRight();

///////////////////////////////////////////////////////////////////////////////////

enum PLATFORM
{
	PLATFORM_WINDOWS, 
	PLATFORM_DOS, 
	PLATFORM_HPUX, 
	PLATFORM_SUNOS, 
	PLATFORM_BASIC
};

enum TYPE
{
	TYPE_IMPLICITI, 
	TYPE_EXPLICITI
};

enum STORE_METHOD
{
	STORE_METHOD_CHANNEL, 
	STORE_METHOD_BLOCK
};

enum DATATYPE
{	
	DATATYPE_INT16, 
	DATATYPE_INT32,
	DATATYPE_WORD8, 
	DATATYPE_WORD16, 
	DATATYPE_WORD32,
	DATATYPE_TWOC12, 
	DATATYPE_TWOC16,
	DATATYPE_REAL32, 
	DATATYPE_REAL48, 
	DATATYPE_REAL64,
	DATATYPE_MSREAL32, 
	DATATYPE_ASCII
};

enum MONOTONE
{	
	MONOTONE_NOT, 
	MONOTONE_NOTCALCULATED,
	MONOTONE_INCREASING, 
	MONOTONE_DECREASING
};

enum LABELTYPE
{
	LABELTYPE_FILE_EMPTY_STRING,
	LABELTYPE_FILE_VALID_FLAG,
	LABELTYPE_FILE_GENERAL_HEADER,
	LABELTYPE_FILE_CHANNEL_HEADER
};

enum GENERALHEADER
{
	ORIGIN						= 1,
	REVISION					= 2,
	DESCRIPTION					= 101,
	FILE_COMMENTS				= 102,
	PERSON						= 103,	
	DATE						= 104,
	TIME						= 105,
	DESCRIPTION_FOR_COMMENTS	= 106,	
	TIME_FORMAT					= 110,
	MISSING_VALUE				= 111,
	BYTE_ORDER					= 112,
};

enum CHANNELHEADER
{
	CHANNEL_NAME				= 200,
	CHANNEL_COMMENTS			= 201,
	UNIT						= 202,
	CHANNEL_TYPE				= 210,
	FILE_NAME					= 211,	
	CHANNEL_STORE_METHOD		= 213,
	DATA_TYPE					= 214,
	BIT_MASKING					= 215,
	LENGTH						= 220,
	FILE_OFFSET					= 221,
	CHANNEL_OFFSET				= 222,
	ASC_POINTER					= 223,
	SEPERATOR					= 230,
	DECIMAL						= 231,
	EXPONENTIAL_SYMBOL			= 232,
	VALUE_START_OR_OFFSET		= 240,
	VALUE_STEP_OR_FACTOR		= 241,
	MIN							= 250,
	MAX							= 251,
	CHANNEL_MISSING_VALUE_FLAG	= 252,
	MONOTONE_FLAG				= 253,
	CHANNEL_MISSING_VALUE		= 254,
	DISPLAY_FORMAT				= 260,	
};

//////////////////////////////////////////////////////

static vector<string> gvsDelimited = {"	", ",", " ", ";"}; 
static char gchDelimiteds[] = {'	', ',', ' ', ';'};

//static int _switch_to_idelimiter(int nChDelimiterIndex)
//{
	//int iDelimeter = DMT_COMMA_INDEX;
	//switch(nChDelimiterIndex)
	//{
	//case SEP_TAB_INDEX:
		//iDelimeter = DMT_TAB_INDEX;
		//break;
	//case SEP_COMMA_INDEX:
		//iDelimeter = DMT_COMMA_INDEX;
		//break;
	//case SEP_SPACE_INDEX:
		//iDelimeter = DMT_SPACE_INDEX;
		//break;
	//case SEP_SEMINAL_INDEX:
		//iDelimeter = DMT_SEMINAL_INDEX;
		//break;
	//case SEP_OTHER_INDEX:
		//iDelimeter = DMT_OTHER_INDEX;
		//break;
	//default:
	//}
	//return iDelimeter;
//}

static void get_seprator_by_ASC(char* chDelimiters, int nSep)
{
	switch(nSep)
	{
	case CHAR_COMMA:
		chDelimiters[SEP_COMMA_INDEX] = CHAR_COMMA;
		break;
	case CHAR_TAB:
		chDelimiters[SEP_TAB_INDEX] = CHAR_TAB;
		break;
	case CHAR_SPACE:
		chDelimiters[SEP_SPACE_INDEX] = CHAR_SPACE;
		break;
	case CHAR_SEMINAL:
		chDelimiters[SEP_SEMINAL_INDEX] = CHAR_SEMINAL;
		break;
	default:
		chDelimiters[SEP_OTHER_INDEX] = nSep;
	}
}

static int _get_sep_index(char* cDelimiters)
{
	int nSepIndex = 0;
	for(int ii = 0; ii < DELIMETERS_ARRAY_SIZE; ii++)
	{
		if(cDelimiters[ii] == '\0')
			continue;
		
		nSepIndex = ii;
		break;
	}
	return nSepIndex;
}

static void _prepare_ascimp_before_import(ASCIMP& ascimp, char* chDelimiters, int nSepIndex)
{
	ascimp.iPartialC2 		= -1;
	ascimp.iPartialR2 		= -1;
	ascimp.nLongNames		= -1;
	ascimp.nUnits			= -1;
	ascimp.nFirstParams		= -1;
	ascimp.nFirstUserParams	= -1;
	
	//int nPos = 4;
	//int nCount = 0;
	//for(int ii = 0; ii < SEP_OTHER_INDEX; ii++)
	//{
		//if(ascimp.cDelimiters[ii] != '\0')
		//{
			//nPos = ii;
			//nCount++;
		//}
	//}
	
	//only one sep
	//if(nCount == 1 || nPos == 4)
	//{
	//ascimp.iDelimiter = _switch_to_idelimiter(nPos);
	/// Hong 12/07/07 v8.0763 FIX_DIADEM_BINARY_FAIL_IMPORT_CAUSED_BY_MARCO_DUPLICATE
	//ascimp.iDelimiter = DMT_SEMINAL_OR_OTHER_VAL;
	ascimp.iDelimiter = ASCIMP_DELIM_OTHER;
	/// end FIX_DIADEM_BINARY_FAIL_IMPORT_CAUSED_BY_MARCO_DUPLICATE
	//if(nSepIndex == SEP_SEMINAL_INDEX)
		//ascimp.cChar = chDelimiters[nSepIndex];
	//else if(nPos == SEP_OTHER_INDEX)
	ascimp.cChar = chDelimiters[nSepIndex]; 
	//}
}

static int _handle_channel_separator(char* chDelimiters, char* chDelimitersArray, TreeNode& trSeperator)
{
	if( !is_missing_value(trSeperator.dVal) )  //ASCII code
	{
		get_seprator_by_ASC(chDelimiters, trSeperator.nVal);
	}
	else //string
	{
		if(trSeperator.strVal.IsEmpty()) //default is comma
			chDelimiters[SEP_COMMA_INDEX] = CHAR_COMMA;
		else
		{
			string strSep = trSeperator.strVal;
			int nLength = strSep.GetLength();
			if(nLength > 1 && nLength < 6) //use string as sep, not more than 5 char
			{
				memcpy(chDelimitersArray, &strSep, nLength);
				return DIADEM_NO_ERROR;
			}
			else if(nLength >= 6) //more than 5, error
				return DIADEM_ERR_UNSUPPORT_FILE;
			
			//use one char as sep
			int nPos = gvsDelimited.Find(strSep);
			if(nPos > 0)
				chDelimiters[nPos] = gchDelimiteds[nPos];
			else
				chDelimiters[SEP_OTHER_INDEX] = trSeperator.strVal.GetAt(0); //support sep with 1 char
		}
	}
	
	return DIADEM_NO_ERROR;
}


//////////////////////////////////////////////////////
////	class DiademFile	
////

DiademFile::DiademFile(LPCTSTR lpszFileName, bool bImport) 
:stdioFile(lpszFileName, file::modeRead | file::shareDenyWrite)	
{
	m_strDatFileName	= lpszFileName;
	m_nFileSize = SeekToEnd();
	m_nBlockChannels = 0;
	///Cheney 2007-1-8 SUPPORT_UNICODE_FILE
	int nUnicodeType = CheckUniCodeBOM();
	///end SUPPORT_UNICODE_FILE
	if(nUnicodeType == UNICODE_NON) //ignore bom
		SeekToBegin();
	ASSERT(bImport);  //ot support writing now
}

DiademFile::~DiademFile()
{
}

BOOL DiademFile::Open(LPCTSTR lpszFileName)
{
	if ( Open(lpszFileName, file::modeRead | file::typeText | file::shareDenyWrite) )
	{
		return true;
	}
	return false;
}

/// Hong 07/17/08 v8.0903 FIX_SOME_IMPORT_HAVE_DUPLICATE_SHEET_NAME_NO_AUTO_ENUM
//int DiademFile::Import(Worksheet& wks, TreeNode& trFileNode, TreeNode& trInfo, bool bReadAsBinary, int nMode, int nC1, DWORD dwOption)
int DiademFile::Import(Worksheet& wks, TreeNode& trFileNode, TreeNode& trInfo, bool bReadAsBinary, int nMode, int nC1, DWORD dwOption) // = 0, DIADEM_RENAME_SHEET_BY_FILENAME
/// end FIX_SOME_IMPORT_HAVE_DUPLICATE_SHEET_NAME_NO_AUTO_ENUM
{
	///---Sim 05-28-2007 QA80-9828-P3 REPEAR_IMPORT_NEED_ALLOW_PASS_DATA_SELECTION
	Tree trFileNodeTemp;	
	int nRet;
	//int nRet = ReadHeader(trFileNode, trInfo);
	nRet = ReadHeader(trFileNodeTemp, trInfo);
	if(nRet)
		return nRet;
	
	if( !(trFileNode && trFileNode.GetNodeCount() > 0) ) // header info no ready
	{
		trFileNode = trFileNodeTemp;
	}	
	///---END QA80-9828-P3 REPEAR_IMPORT_NEED_ALLOW_PASS_DATA_SELECTION
	
	if(nRet = ReadData(wks, trFileNode, bReadAsBinary, nMode, nC1, dwOption))
		return nRet;
	
	/// Hong 07/17/08 v8.0903 FIX_SOME_IMPORT_HAVE_DUPLICATE_SHEET_NAME_NO_AUTO_ENUM
	//wks.SetName(GetFileName(m_strDatFileName, TRUE));
	if ( DIADEM_RENAME_SHEET_BY_FILENAME & dwOption )
		wks.SetName(GetFileName(m_strDatFileName, TRUE), OCD_ENUM_NEXT | OCD_ENUM_ADD_SEPARATOR);
	/// end FIX_SOME_IMPORT_HAVE_DUPLICATE_SHEET_NAME_NO_AUTO_ENUM
	return DIADEM_NO_ERROR;
}

int DiademFile::ReadHeader(TreeNode& trFileNode, TreeNode& trInfo)
{	
	if(!IsOpen())
	{
		Close();
		return DIADEM_ERR_INVALID_FILE;
	}
	
	m_trHeader = trInfo.AddNode(STR_HEADER_TAG_NAME);
	m_trChannels = trInfo.AddNode(STR_CHANNELS_TAG_NAME);
	if ( LABELTYPE_FILE_VALID_FLAG == getRecord(trFileNode))   // first line must be LABELTYPE_FILE_VALID_FLAG
	{
		while ( getRecord(trFileNode) >= LABELTYPE_FILE_EMPTY_STRING && GetPosition() < m_nFileSize)
		{
		}
	}
	else
	{
		Close();
		return DIADEM_ERR_INVALID_FILE;
	}
	
	Close();
	trInfo = m_trHeader; //delete m_trChannels, just general header should show orgnize
	return DIADEM_NO_ERROR;
}

int DiademFile::getRecord(TreeNode& trFileNode)
{
	int nRet = LABELTYPE_FILE_EMPTY_STRING;
	string strLine;
	if(ReadString(strLine))
	{
		TRIM_STRING_LEFT_RIGHT_SPACE(strLine)
		if(strLine == STR_DIADEM_VALID_FLAG)
		{
			nRet = LABELTYPE_FILE_VALID_FLAG;
		}
		else if(strLine == STR_DIADEM_GENERAL_HEADER_BEGIN)
		{
			nRet = getGeneralRecord();
		}
		else if(strLine == STR_DIADEM_CHANNEL_HEADER_BEGIN)
		{
			nRet = getChannelRecord(trFileNode);
		}
	}
 
	return nRet;
}

/// Hong 04/14/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
static void _cvt_Diadem_time_fmt_to_OC(string& strTimeFormat)
{
	strTimeFormat.Replace('m', 'M');
	strTimeFormat.Replace('h', 'H');
	strTimeFormat.Replace('n', 'm');
	if ( strTimeFormat.Replace('f', '#') > 0 )
	{
		int		nPos = strTimeFormat.Find('#');
		ASSERT(-1 != nPos);
		if ( nPos > 0 && '.' == strTimeFormat[nPos - 1] )
		{
			strTimeFormat.Insert(nPos, '\'');
			strTimeFormat.Insert(nPos - 1, '\'');
		}
	}
}
/// end FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED

int DiademFile::getGeneralRecord()
{
	int nRet = LABELTYPE_FILE_GENERAL_HEADER;
	string strLine;
			
	TreeNode trCurrentNode; //DESCRIPTION_FOR_COMMENTS and COMMENTS will be a pair
							//DESCRIPTION_FOR_COMMENTS will be node's name, COMMENTS will be node's value
	int nCommentCount = 0;
	while(ReadString(strLine))
	{
		TRIM_STRING_LEFT_RIGHT_SPACE(strLine)
		if(strLine == STR_DIADEM_GENERAL_HEADER_END)
			break;
		
		int nKeyWord = atof(strLine.GetToken(0, CHAR_COMMA));
		int nPos = strLine.Find(CHAR_COMMA);
		string strKeyValue = strLine.Right(strLine.GetLength() - nPos - 1);
		
		switch(nKeyWord)
		{
		case ORIGIN:
			m_trHeader.AddTextNode(strKeyValue, "Origin");
			break;
		case REVISION:
			m_trHeader.AddTextNode(strKeyValue, "Revision");
			break;
		case DESCRIPTION:
			m_trHeader.AddTextNode(strKeyValue, "DescriptionOfDataset");
			break;
		case FILE_COMMENTS:
			///Cheney 2007-1-8 SUPPORT_UNICODE_FILE
			//if(!strKeyValue.IsEmpty())
				//trCurrentNode.strVal = strKeyValue;
			//else
				//trCurrentNode.Remove();
			if(strKeyValue.IsEmpty() && trCurrentNode.strVal.IsEmpty())
				trCurrentNode.Remove();
			else
				m_trHeader.AddTextNode(strKeyValue, "CommentsOnDataset");
			///end SUPPORT_UNICODE_FILE
			break;
		case PERSON:
			m_trHeader.AddTextNode(strKeyValue, "Author");
			break;
		case DATE:
			m_trHeader.AddTextNode(strKeyValue, "Date");
			break;
		case TIME:
			m_trHeader.AddTextNode(strKeyValue, "Time");
			break;
		case DESCRIPTION_FOR_COMMENTS:
			///Cheney 2007-1-8 SUPPORT_UNICODE_FILE
			//nCommentCount++;
			trCurrentNode = m_trHeader.AddTextNode(strKeyValue, "DescriptionOfComments");
			//if(!is_missing_value(atof(strKeyValue)) )
			//	strKeyValue = "_" + strKeyValue; //neumerical can not set as node's tag name
			//trCurrentNode = m_trHeader.AddNode(strKeyValue.IsEmpty()? "DataSetComment"+ftoa(nCommentCount) : strKeyValue);
			///end SUPPORT_UNICODE_FILE
			break;
		case TIME_FORMAT:
			strKeyValue.TrimLeft('#');
			/// Hong 04/14/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
			_cvt_Diadem_time_fmt_to_OC(strKeyValue);
			/// end FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
			m_trHeader.AddTextNode(strKeyValue, "TimeFormat");
			break;
		case MISSING_VALUE:
			m_trHeader.AddTextNode(strKeyValue, "MissingValue");
			break;
		case BYTE_ORDER:
			m_trHeader.AddTextNode(strKeyValue, "ByteOrder");
			m_bLittleEndian = false;
			if(strKeyValue != "High -> Low")
				m_bLittleEndian = true;
			break;
		default:
			return DIADEM_ERR_INVALID_LABEL;
		}
	}
	return nRet;
}

int DiademFile::getChannelRecord(TreeNode& trFileNode)
{
	int nRet = LABELTYPE_FILE_CHANNEL_HEADER;
	
	TreeNode trChannel = m_trChannels.AddNode(STR_CHANNEL_TAG_NAME);
	string strLine;
	bool bHasChannelName = false; //Channel should has name, otherwise invalid
	while(ReadString(strLine))
	{
		TRIM_STRING_LEFT_RIGHT_SPACE(strLine)
		if(strLine == STR_DIADEM_CHANNEL_HEADER_END)
			break;
		
		int nKeyWord = atof(strLine.GetToken(0, CHAR_COMMA));
		int nPos = strLine.Find(CHAR_COMMA);
		string strKeyValue = strLine.Right(strLine.GetLength() - nPos - 1);
		
		switch(nKeyWord)
		{
		case CHANNEL_NAME:
			trChannel.AddTextNode(strKeyValue, "ChannelName");
			TreeNode trTemp = trFileNode.AddNode("ChannelSetting");
			trTemp.SetAttribute(STR_LABEL_ATTRIB, strKeyValue);
			bHasChannelName = true;
			break;
		case CHANNEL_COMMENTS:
			trChannel.AddTextNode(strKeyValue, "ChannelComments");
			break;
		case UNIT:
			trChannel.AddTextNode(strKeyValue, "Unit");
			break;
		case CHANNEL_TYPE:
			trChannel.AddTextNode(strKeyValue, "ChannelType");
			break;
		case FILE_NAME:
			trChannel.AddTextNode(strKeyValue, "FileName");
			break;
		case CHANNEL_STORE_METHOD:
			trChannel.AddTextNode(strKeyValue, "StoreMethod");
			break;
		case DATA_TYPE:
			trChannel.AddTextNode(strKeyValue, "DataType");
			break;
		case BIT_MASKING:
			trChannel.AddTextNode(strKeyValue, "BitMasking");
			break;
		case LENGTH:
			trChannel.AddTextNode(strKeyValue, "Length");
			break;
		case FILE_OFFSET:
			trChannel.AddTextNode(strKeyValue, "FileOffset");
			break;
		case CHANNEL_OFFSET:
			trChannel.AddTextNode(strKeyValue, "ChannelOffset");
			break;
		case ASC_POINTER:
			trChannel.AddTextNode(strKeyValue, "ASCPointer");
			break;
		case SEPERATOR:
			trChannel.AddTextNode(strKeyValue, "Seperator");
			break;
		case DECIMAL:
			trChannel.AddTextNode(strKeyValue, "Decimal");
			m_bCommaDecimal = strKeyValue==STR_COMMA || (char)atof(strKeyValue) == CHAR_COMMA? true : false;
			m_bBinaryString = strKeyValue=="B" || strKeyValue=="b" || (char)atof(strKeyValue) == 'B' || (char)atof(strKeyValue) == 'b'? true : false;
			if(strKeyValue=="H" || strKeyValue=="h" || (char)atof(strKeyValue) =='H' || (char)atof(strKeyValue) == 'h')  //"h" we don't support it
				return DIADEM_ERR_UNSUPPORT_FILE;
			break;
		case EXPONENTIAL_SYMBOL:
			trChannel.AddTextNode(strKeyValue, "ExponentialSymbol");
			break;
		case VALUE_START_OR_OFFSET:
			trChannel.AddTextNode(strKeyValue, "ValueStartOrOffset");
			break;
		case VALUE_STEP_OR_FACTOR:
			trChannel.AddTextNode(strKeyValue, "ValueStepOrFactor");
			break;		
		case MIN:
			trChannel.AddTextNode(strKeyValue, "Min");
			break;		
		case MAX:
			trChannel.AddTextNode(strKeyValue, "Max");
			break;
		case CHANNEL_MISSING_VALUE_FLAG:
			trChannel.AddTextNode(strKeyValue, "MissingValueFlag");
			break;	
		case MONOTONE_FLAG:
			trChannel.AddTextNode(strKeyValue, STR_MONOTONE_FLAG);
			break;
		case CHANNEL_MISSING_VALUE:
			trChannel.AddTextNode(strKeyValue, "MissingValue");
			break;
		case DISPLAY_FORMAT:
			trChannel.AddTextNode(strKeyValue, "DisplayFormat");
			break;
		default: //ignore
		}
	}
	if(!bHasChannelName)
		nRet = DIADEM_ERR_LACK_CHANNEL_NAME;
	
	return nRet;
}

int DiademFile::getDataType(TreeNode& trDataType)
{
	if(!trDataType)
		return DIADEM_ERR_INVALID_DATA_TYPE;
	
	string strDataType = trDataType.strVal;
	if(strDataType.IsEmpty())
		return DIADEM_ERR_INVALID_DATA_TYPE;
	
	int nRet = DIADEM_ERR_INVALID_DATA_TYPE;
	if(strDataType == "INT16")
		nRet = DATATYPE_INT16;
	else if(strDataType=="INT32")
		nRet = DATATYPE_INT32;
	else if(strDataType=="WORD8") 
		nRet = DATATYPE_WORD8;
	else if(strDataType=="WORD16")
		nRet = DATATYPE_WORD16;
	else if(strDataType=="WORD32")
		nRet = DATATYPE_WORD32;
	else if(strDataType=="REAL32")
		nRet = DATATYPE_REAL32;	
	else if(strDataType=="REAL48")
		nRet = DATATYPE_REAL48;	
	else if(strDataType==STR_DATA_TYPE_REAL_64)
		nRet = DATATYPE_REAL64;	
	else if(strDataType=="MSREAL32")
		nRet = DATATYPE_MSREAL32;	
	else if(strDataType=="TWOC12")
		nRet = DATATYPE_TWOC12;
	else if(strDataType=="TWOC16")
		nRet = DATATYPE_TWOC16;
	else if(strDataType=="ASCII")
		nRet = DATATYPE_ASCII;
	
	return nRet;
}

WORD DiademFile::sizeOf()
{
	WORD wSize = 0;
	switch (m_nChannelDatatype)
	{
		case DATATYPE_TWOC12:
		case DATATYPE_TWOC16:
			wSize = 0;
			break;
		case DATATYPE_WORD8:
			wSize = 1;
			break;
		case DATATYPE_INT16:
		case DATATYPE_WORD16:
			wSize = 2;
			break;
		case DATATYPE_INT32:
		case DATATYPE_WORD32:
		case DATATYPE_REAL32:
		case DATATYPE_MSREAL32:
			wSize = 4;
			break;
		case DATATYPE_REAL48:
			wSize = 6;
			break;
		case DATATYPE_REAL64:
			wSize = 8;
			break;
	default:
	}
	return wSize;

}

int DiademFile::ReadData(Worksheet& wks, TreeNode& trFileNode, bool bReadAsBinary, int nMode, int nC1, DWORD dwOption)
{
	m_strDataFilePath = GetFilePath(m_strDatFileName);
	if(!m_strDataFilePath.IsPath())
		return CER_INVALID_FILEPATH;
	
	//check store method, no mix, means channel or block(and only one block)
	int nStoreMethod = STORE_METHOD_CHANNEL;
	int nChannelIndex = 0;
	///---Sim 04-18-2007 ADD_RANGE_INFO
	//int nTotalCols = nC1;
	int nNumCols = 0;
	
	if ( m_orng )
		m_orng.Reset();
	///---END ADD_RANGE_INFO
	
	//if ASCII data, all channel explicit
	//just for ASCII and Block store, can specify separator and desimal
	//and it also can specify Expotional symbol, impasc can handle it already.
	//seems only one kind desimal, separator, expotional symbol in ASCII data file
	//only support sep with 1 char
	Tree trOptions; //set a default filter for ASC import
	fuPrepareDefault(trOptions, FILTER_TYPE_ASCII);
	ASCIMP ascimp;
	ascimp = trOptions.ASCIMP;
	/// Hong 07/11/07 QA9828-P4 v8.0657 FIX_IMPORT_AS_BINARY_REIMPORT_UNWANTED_MISSING_VALUE
	ascimp.iAutoSubHeaderLines = 0; // cheney said no header line in data file, so should disable auto
	ascimp.iLeadingZeroes = 1;
	/// end FIX_IMPORT_AS_BINARY_REIMPORT_UNWANTED_MISSING_VALUE
	char cDelimiters[DELIMETERS_ARRAY_SIZE] = '\0';
	/// Hong 07/18/07 QA80-9828-P6 REIMPORT_NOT_SUPPORT_SOURCE_FILE_CHANGE_CHANNEL_NUM
	if ( m_trChannels.GetNodeCount() != trFileNode.GetNodeCount() )
		return DIADEM_ERR_UNSUPPOORT_CHANNEL_CHANGE;
	/// end REIMPORT_NOT_SUPPORT_SOURCE_FILE_CHANGE_CHANNEL_NUM
	bool bGetChannelDatatype = false;
	foreach (TreeNode trChannel in m_trChannels.Children )
	{
		TreeNode trChannelSetting = trFileNode.Children.Item(nChannelIndex++);
		if(trChannelSetting.nVal == 1) // if this channel in trSetting not check, continue
		///---Sim 04-18-2007 ADD_RANGE_INFO
			//nTotalCols++;
			nNumCols++;
		///---END ADD_RANGE_INFO
		
		if(trChannel.StoreMethod) //only explicit store, the node will exist
		{
			if(!bGetChannelDatatype)
			{
				m_nChannelDatatype = getDataType(trChannel.DataType);
				bGetChannelDatatype = true;
			}
			
			nStoreMethod = trChannel.StoreMethod.strVal == "CHANNEL"? STORE_METHOD_CHANNEL : STORE_METHOD_BLOCK;
			if(nStoreMethod == STORE_METHOD_BLOCK)
				m_nBlockChannels++;
		}
	}
	
	///---Sim 04-18-2007 ADD_RANGE_INFO
	m_orng.Add(wks, nC1, "Range", nC1 + nNumCols - 1);
	///---END ADD_RANGE_INFO
	
	int nRet = DIADEM_NO_ERROR;
	
	//just for ASCII and Block store, only one sep, should prepare it(refer to impwiz)
	//handle file's separator, just for ASCII
	TreeNode trFirstChannel = m_trChannels.Children.Item(0);
	bool bASCAndBlock = getDataType(trFirstChannel.DataType) == DATATYPE_ASCII && nStoreMethod != STORE_METHOD_CHANNEL;
	if(bASCAndBlock)
	{
		if(trFirstChannel.Seperator)
		{
			if(nRet = _handle_channel_separator(cDelimiters, ascimp.cDelimitersArray, trFirstChannel.Seperator))
				return nRet;
		}
		
		int nSepIndex = _get_sep_index(cDelimiters);
		//if(nSepIndex == SEP_SEMINAL_INDEX || nSepIndex == SEP_OTHER_INDEX) //otherwise, use default
		_prepare_ascimp_before_import(ascimp, cDelimiters, nSepIndex);
		
		if(m_bBinaryString && !bReadAsBinary)
			ascimp.nSpeFmt = ASC_IMP_BIN_FORMAT;
		
		if(m_bCommaDecimal)
			ascimp.nNumSep = NF_IS_EUROPEAN;   //use it to handle decimal as comma
		
		ascimp.iAutoColTypes = 2; //detect col types
		ascimp.iNonnumeric = 1; //handle non_numeric in numeric field
		
		//handle date data
		string strTimeFormat = m_trHeader.TimeFormat.strVal;
		memcpy(ascimp.szDateFormat, &strTimeFormat, strTimeFormat.GetLength());
	}
	
	int nColIndex = nC1;
	nChannelIndex = 0;
	int nCount = m_trChannels.Children.Count();
	int nCurrentPos = 0; //for read ASC data when channel-wise store
	
	//if block-wise and ASCII, and separator is not CRLF, will call imp ASC
	//otherwise, ASCII will read as channel
	bool bReadAscBlock = bASCAndBlock && trFirstChannel.Seperator && trFirstChannel.Seperator.strVal != STR_CRLF;
	if(bReadAscBlock)
	{
		//if wks has not enough col, should add it
		///---Sim 04-18-2007 ADD_RANGE_INFO
		//nTotalCols = m_trChannels.Children.Count() + nC1;
		nNumCols = m_trChannels.Children.Count();
		
		/// Hong 05/25/07 v8.0628 FIX_RUNTIME_ERROR_BY_ERROR_RANGE
		// no need update as we have do caculation to get the correct import range
		//// update data range
		//if ( m_orng )
			//m_orng.Reset();
		//m_orng.Add(wks, nC1, "Range", nC1 + nNumCols - 1);
		/// end FIX_RUNTIME_ERROR_BY_ERROR_RANGE
		
		//if(nTotalCols > wks.GetNumCols())
			//wks.SetSize(-1, nTotalCols);
		if(nC1 + nNumCols > wks.GetNumCols())
			wks.SetSize(-1, nC1 + nNumCols);
		///---END ADD_RANGE_INFO
		//for(int nLoop = wks.GetNumCols(); nLoop < nTotalCols; nLoop = wks.GetNumCols() ) 
			//wks.AddCol();
		
		ascimp.iMode = nMode;
		/// Hong 07/20/07 QA80-9828-P7 v8.0664 FIX_REIMPORT_BINARY_DIADEM_DUPLICATE_STORAGE_INFO
		//int nRet = wks.ImportASCII(m_strDataFilePath+trChannel.FileName.strVal, ascimp); 
		int nRet = wks.ImportASCII(m_strDataFilePath+trChannel.FileName.strVal, ascimp, NULL, NULL, IMPASC_NOT_SAVE_STORAGE_INFO); 
		/// end FIX_REIMPORT_BINARY_DIADEM_DUPLICATE_STORAGE_INFO
		for(int ii = 0; ii < nCount; ii++)
		{
			trChannel = m_trChannels.Children.Item(ii);
			TreeNode trChannelSetting = trFileNode.Children.Item(nChannelIndex++);
			if(trChannelSetting.nVal != 1) // if this channel in trSetting not check, continue
				wks.DeleteCol(nColIndex);
			else
			{
				Dataset ds;
				/// Hong 04/16/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
				//nRet = handleColumnInfo(wks, ds, trChannel, nColIndex, nChannelIndex);
				nRet = handleColumnInfo(wks, ds, trChannel, nColIndex, nChannelIndex, false);
				/// end FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
				if(nRet != DIADEM_NO_ERROR)
					return nRet;
	
				nColIndex++;	
			}
		}
	}
	else //not ASCII and Block store
	{
		//if wks has not enough col, should add it
		//for(int nLoop = wks.GetNumCols(); nLoop < nTotalCols; nLoop = wks.GetNumCols() ) 
			//wks.AddCol();
		///---Sim 04-18-2007 ADD_RANGE_INFO
		//if(nTotalCols > wks.GetNumCols())
			//wks.SetSize(-1, nTotalCols);
		if(nC1 + nNumCols > wks.GetNumCols())
			wks.SetSize(-1, nC1 + nNumCols);
		///---END ADD_RANGE_INFO
		
		//m_nChannelDatatype = getDataType(trChannel.DataType);
		
		/// Hong 08/04/10 ORG-635-P1 FIX_BLOCK_DATA_FAIL_IMPORT_WHEN_STORED_IN_MULTI_FILES
		// Hong, for block data, it can store in multi-files when they have diff length
		/*
		//if store as block, all explicit data will be store in a block, should read it at first
		void* pData;
		if(nStoreMethod == STORE_METHOD_BLOCK)
		{
			int nBufLength = trChannel.Length.nVal*m_nBlockChannels;
			DWORD dwDataSize = sizeOf();
			pData = calloc(nBufLength, dwDataSize);
			
			int nFileOffset = 0;
			if(nRet = diadem_read_explicit_data(pData, nBufLength, m_strDataFilePath+trChannel.FileName.strVal, 
					 nFileOffset, STORE_METHOD_BLOCK, m_nChannelDatatype, dwDataSize))
			{
				free(pData);
				return nRet;
			}
		}
		*/
		void* 		pData = NULL;
		string		strOldBlockStoredFile;
		/// end FIX_BLOCK_DATA_FAIL_IMPORT_WHEN_STORED_IN_MULTI_FILES
	
		//get data
		int nBlockChannelIndex = 0;  //for recode channel index in block 
		for(int ii = 0; ii < nCount; ii++)
		{
			trChannel = m_trChannels.Children.Item(ii);
			if(!trChannel.ChannelType)
				return DIADEM_ERR_INVALID_FILE;
			
			///Cheney 2007-3-7 DATASET_NOW_ONLY_SUPPORT_DOUBLE_TYPE_COLUMN			
			wks.Columns(nColIndex).SetFormat(OKCOLTYPE_NUMERIC);
			wks.Columns(nColIndex).SetInternalDataType(FSI_DOUBLE);
			///end DATASET_NOW_ONLY_SUPPORT_DOUBLE_TYPE_COLUMN
			
			Dataset ds;
			if(trChannel.ChannelType.strVal == STR_DATA_STORE_IMPLICIT) //implicit
			{
				TreeNode trChannelSetting = trFileNode.Children.Item(nChannelIndex++);
				if(trChannelSetting.nVal != 1) // if this channel in trSetting not check, continue
					continue;
				
				ds.Attach(wks, nColIndex);
				nRet = getImplicitData(ds, trChannel);
				if(nRet != DIADEM_NO_ERROR)
					return nRet;
				
				nRet = handleColumnInfo(wks, ds, trChannel, nColIndex, nChannelIndex);
				if(nRet != DIADEM_NO_ERROR)
					return nRet;
	
				nColIndex++;
			}
			else //explicit
			{
				//m_nChannelDatatype = getDataType(trChannel.DataType);
				handelFactorAndOffset(trChannel);
				
				if(m_nChannelDatatype != DATATYPE_ASCII)
				{
					if(nStoreMethod == STORE_METHOD_CHANNEL)
					{
						nRet = getExplicitDataChannel(wks, ds, nColIndex, nChannelIndex, trChannel.Length.nVal, trChannel, trFileNode);
					}
					else //block
					{
						/// Hong 08/04/10 ORG-635-P1 FIX_BLOCK_DATA_FAIL_IMPORT_WHEN_STORED_IN_MULTI_FILES
						if ( NULL == pData || 0 != strOldBlockStoredFile.CompareNoCase(trChannel.FileName.strVal) )
						{
							if ( NULL != pData )
								free(pData);
							
							m_nBlockChannels = trChannel.ChannelOffset.nVal;
							m_nChannelDatatype = getDataType(trChannel.DataType);
							
							int		nBufLength = trChannel.Length.nVal * m_nBlockChannels;
							DWORD	dwDataSize = sizeOf();
							pData = calloc(nBufLength, dwDataSize);
							
							int 	nFileOffset = 0;
							if(nRet = diadem_read_explicit_data(pData, nBufLength, m_strDataFilePath+trChannel.FileName.strVal, 
									 nFileOffset, STORE_METHOD_BLOCK, m_nChannelDatatype, dwDataSize))
							{
								free(pData);
								return nRet;
							}
						}
						/// end FIX_BLOCK_DATA_FAIL_IMPORT_WHEN_STORED_IN_MULTI_FILES
						/// Hong 08/04/10 ORG-635-P2 FIX_FAIL_IMPORT_CORRECT_CHANNEL_WHEN_SKIP_BLOCK_DATA
						nBlockChannelIndex = trChannel.FileOffset.nVal - 1;
						/// end FIX_FAIL_IMPORT_CORRECT_CHANNEL_WHEN_SKIP_BLOCK_DATA
						nRet = getExplicitDataBlock(pData, wks, ds, nColIndex, nChannelIndex, nBlockChannelIndex, trChannel.Length.nVal, trChannel, trFileNode);
					}
				}
				else //ASCII
				{
					//Channel-wise or block-wise but use CR\LF as separator
					if(!bReadAscBlock)
						nRet = getASCDataChannel(wks, ds, nColIndex, nChannelIndex, trChannel.Length.nVal, &nCurrentPos, trChannel, trFileNode, bReadAsBinary);
				}
			}
			
			if(nRet != DIADEM_NO_ERROR)
			{
				if(nStoreMethod == STORE_METHOD_BLOCK)
					free(pData);
				
				return nRet;
			}
			
			///Cheney 2007-4-21 AFTER_SET_ALL_DATA_NO_NEED_LOOP_CONTINUE
			//if(nColIndex >= nNumCols)
			/// Hong 04/27/07 QA80-9247-P18 FIX_FAIL_WHEN_MULTI_FILE_WITH_START_NEW_COLUMN
			if(nColIndex >= nC1 + nNumCols)
			/// end FIX_FAIL_WHEN_MULTI_FILE_WITH_START_NEW_COLUMN
				break;
			///end AFTER_SET_ALL_DATA_NO_NEED_LOOP_CONTINUE
		}
		
		if(nStoreMethod == STORE_METHOD_BLOCK)
			free(pData);
	}
	
	return nRet;
}

int DiademFile::getImplicitData(vector& vData, TreeNode& trChannel)
{
	if(!trChannel.ValueStartorOffset || !trChannel.ValueSteporFactor)
		return DIADEM_ERR_INVALID_FILE;
	
	double dStart	= trChannel.ValueStartorOffset.dVal;
	double dStep	= trChannel.ValueSteporFactor.dVal;
	LONG nLength	= trChannel.Length.nVal;
	double dEnd		= dStart + (nLength-1) * dStep;
	
	int nRet = DIADEM_NO_ERROR;
	///Cheney 2007-1-9 HANDLE_MONOTONE_OF_IMPLICIT_DATA
	bool bDecreasing = false;
	///end HANDLE_MONOTONE_OF_IMPLICIT_DATA
	
	vData.SetSize(nLength);
	
	if(dStart == dEnd)
	{
		vData = dStart;
		///Cheney 2007-1-9 HANDLE_MONOTONE_OF_IMPLICIT_DATA
		bDecreasing = true;
		///end HANDLE_MONOTONE_OF_IMPLICIT_DATA
	}
	else if(!vData.Data(dStart, dEnd, dStep))
		nRet = DIADEM_ERR_EXIST_ILLEGAL_LABEL_VAL;
	
	///Cheney 2007-1-9 HANDLE_MONOTONE_OF_IMPLICIT_DATA
	trChannel.AddTextNode(bDecreasing? "decreasing" : "increasing", STR_MONOTONE_FLAG);
	///end HANDLE_MONOTONE_OF_IMPLICIT_DATA
	
	return nRet;
}

int DiademFile::getExplicitDataChannel(Worksheet& wks, Dataset& ds, int& nColIndex, int& nChannelIndex, int nColLength, TreeNode& trChannel, TreeNode& trFileNode)
{
	TreeNode trChannelSetting = trFileNode.Children.Item(nChannelIndex++);
	if(trChannelSetting.nVal != 1) // if this channel in trSetting not check, continue
		return DIADEM_NO_ERROR;
			
	int nBufLength = nColLength;
	DWORD dwDataSize = sizeOf();
	void* pData = calloc(nBufLength, dwDataSize);
	int nRet = DIADEM_NO_ERROR;
	
	if(nRet = diadem_read_explicit_data(pData, nBufLength, m_strDataFilePath+trChannel.FileName.strVal, 
			 trChannel.FileOffset.nVal, STORE_METHOD_CHANNEL, m_nChannelDatatype, dwDataSize))
	{
		free(pData);
		return nRet;
	}
	
	string strColRange;
	make_column_range_string(strColRange, wks.Columns(nColIndex));
	nRet = diadem_get_explicit_data_channel(strColRange, pData, nBufLength, nColLength, m_nChannelDatatype, 
			dwDataSize, m_dFactor, m_dOffset, m_bLittleEndian);			
	free(pData);
	
	if(nRet != DIADEM_NO_ERROR)
		return nRet;
	
	return handleColumnInfo(wks, ds, trChannel, nColIndex++, nChannelIndex);
}

int DiademFile::getExplicitDataBlock(void* pData, Worksheet& wks, Dataset& ds, int& nColIndex, int& nChannelIndex, int& nBlockChannelIndex, int nColLength, TreeNode& trChannel, TreeNode& trFileNode)
{	
	int nRet = DIADEM_NO_ERROR;
		
	TreeNode trChannelSetting = trFileNode.Children.Item(nChannelIndex++);
	if(trChannelSetting.nVal != 1) // if this channel in trSetting not check, continue
		return nRet;

	string strColRange;
	make_column_range_string(strColRange, wks.Columns(nColIndex));
	nRet = diadem_get_explicit_data_block(strColRange, pData, m_nBlockChannels*nColLength, nColLength, 
								m_nChannelDatatype, sizeOf(), m_dFactor, m_dOffset, 
								m_nBlockChannels, nBlockChannelIndex++, m_bLittleEndian);

	if(nRet != DIADEM_NO_ERROR)
	{
		free(pData);
		return nRet;
	}
	
	nRet = handleColumnInfo(wks, ds, trChannel, nColIndex, nChannelIndex);
	if(nRet != DIADEM_NO_ERROR)
	{
		free(pData);
		return nRet;
	}
	nColIndex++;
	return nRet;
}

int DiademFile::getASCDataChannel(Worksheet& wks, Dataset& ds, int& nColIndex, int& nChannelIndex, int nColLength, int* pCurrentPos, TreeNode& trChannel, TreeNode& trFileNode, bool bReadAsBinary)
{
	TreeNode trChannelSetting = trFileNode.Children.Item(nChannelIndex++);
	bool bReadChannel = true;
	if(trChannelSetting.nVal != 1) // if this channel in trSetting not check, continue
		bReadChannel = false;
	
	//if channle type is date, should try to read it as date data, otherwise read as string
	string strDateFile = m_strDataFilePath+trChannel.FileName.strVal;
	bool bHandelDate = true;
	bool bDateData = trChannel.DisplayFormat.strVal == STR_TIME? true : false;
	if(bDateData)
	{
		stdioFile f;
		if( !f.Open(strDateFile, file::modeRead | file::typeBinary | file::shareDenyWrite)) 
		{
			f.Close();
			return DIADEM_ERR_INVALID_FILE;
		}
		f.Seek(*pCurrentPos, file::begin);
		
		//to loop find the 1st non-missing value
		for(int ii = 0; ii < nColLength; ii++)
		{
			string strLine;
			f.ReadString(strLine);
			
			//missing value
			if(strLine.IsEmpty() || strLine.CompareNoCase("NOVALUE") == 0)
				continue;
			
			double dDate;
			bHandelDate = str_to_date_custom(strLine, m_trHeader.TimeFormat.strVal, &dDate);
			break; //check date finished
		}
		f.Close();
	}
	
	string strColRange;
	if(bReadChannel)
		make_column_range_string(strColRange, wks.Columns(nColIndex));
	
	int nRet = DIADEM_NO_ERROR;
	if(bDateData && !bHandelDate) //read as string
	{
		StringArray saArray(nColLength);
		if(nRet = diadem_get_asc_data_as_string(&saArray, strDateFile, pCurrentPos))
			return nRet;
		
		if(bReadChannel)
		{
			///Cheney 2007-3-7 DATASET_NOW_ONLY_SUPPORT_DOUBLE_TYPE_COLUMN
			wks.Columns(nColIndex).SetFormat(OKCOLTYPE_TEXT);
			///end DATASET_NOW_ONLY_SUPPORT_DOUBLE_TYPE_COLUMN
			wks.Columns(nColIndex).PutStringArray(saArray);
		}
	}
	else //convert to double, if date data, should handle it
	{
		if(nRet = diadem_get_asc_data(strColRange, nColLength, strDateFile, pCurrentPos, bReadChannel, m_bCommaDecimal, m_bBinaryString, bReadAsBinary, bDateData, m_trHeader.TimeFormat.strVal))
			return nRet;
	}
	
	if(bReadChannel)
	{
		/// Hong 04/16/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
		//nRet = handleColumnInfo(wks, ds, trChannel, nColIndex++, nChannelIndex);
		nRet = handleColumnInfo(wks, ds, trChannel, nColIndex++, nChannelIndex, false);
		/// end FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
	}
	return nRet;
}

/// Hong 04/16/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
//void DiademFile::handleDateData(Column& col)
void DiademFile::handleDateData(Column& col, bool bNeedOffsetDateData/* = true*/)
/// end FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
{
	/// Hong 04/14/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
	//Dataset ds(col);
	//ds = ds/DAY_OF_A_YEAR + DIFF_TIME_OFFSET_BETTEEN_TDM_OC;
	//col.SetFormat(OKCOLTYPE_DATE);
	/// Hong 04/16/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
	if ( bNeedOffsetDateData )
	{
		Dataset ds(col);
		ds = ds/DAY_OF_A_YEAR + DIFF_TIME_OFFSET_BETTEEN_TDM_OC;
	}
	/// end FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
	Tree tr;
	if( info_get_section(col, tr, "system.display") )
	{
		tr.CustomFormat.strVal = m_trHeader.TimeFormat.strVal;
		info_set_section(col, tr, "system.display");
	}
	col.SetFormat(OKCOLTYPE_DATE, LDF_OBJ_CUSTOM);
	/// end FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
}

void DiademFile::handelColumnParameter(Column& col, TreeNode& trChannel, int nChannelIndex, int nColIndex)
{
	if(trChannel.ChannelName)
		col.SetLongName(trChannel.ChannelName.strVal);
	if(trChannel.ChannelComments)
		col.SetComments(trChannel.ChannelComments.strVal);
	if(trChannel.Unit)
		col.SetUnits(trChannel.Unit.strVal);
	
	/// Hong 04/13/09 FIX_DUPLICATE_SNAME_AFTER_RENAME_LEAD_TO_COLUMN_MISSING
	// Hong, i dot NOT why this rename is necessary, but it seems safe to remove it as no code dependent on ShortName after import
	//col.SetName("A" + ftoa(nColIndex+1)); //for support trSetting in ASCII and BLOCK store
	/// end FIX_DUPLICATE_SNAME_AFTER_RENAME_LEAD_TO_COLUMN_MISSING
	
	vector<string> vsNames, vsValues;
	vsNames.Add("Missing Value Flag");
	vsNames.Add("Missing Value");
	vsNames.Add("Monotone");
	vsNames.Add("Channel Index");
	
	//if trChannel.MissingValueFlag not exist, should be "IMPLICIT" data type
	if(trChannel.ChannelType.strVal == STR_DATA_STORE_IMPLICIT || !m_bChannelHasMissing)
		vsValues.Add( STR_NO );
	else
		vsValues.Add( STR_YES );
	vsValues.Add(ftoa(m_dChannelMissing));
	trChannel.MonotoneFlag? vsValues.Add(trChannel.MonotoneFlag.strVal) : vsValues.Add(STR_EMPTY);
	vsValues.Add( ftoa(nChannelIndex) );
	
	set_user_parameters(col, vsNames, vsValues);
}

void DiademFile::handelColumnUserInfo(Column& col, TreeNode& trChannel)
{
	string strChannelInfo = "ChannelInfo";
	Tree trChannelInfo;
	trChannelInfo.AddTextNode(m_strDatFileName, "ImportFile");
	trChannelInfo.AddNode(trChannel.DisplayFormat);
	trChannelInfo.AddTextNode(!trChannel.DataType? STR_DATA_TYPE_REAL_64 : trChannel.DataType.strVal, "DataType");
	trChannelInfo.AddTextNode(trChannel.ChannelType.strVal, "Storage");
	trChannelInfo.AddTextNode(!trChannel.FileName? GetFileName(m_strDatFileName) : trChannel.FileName.strVal, "SourceDataFile");
	trChannelInfo.AddTextNode(m_strDataFilePath, "SourceDataFilePath");
	trChannelInfo.AddTextNode(trChannel.ChannelType.strVal!=STR_DATA_STORE_IMPLICIT? ftoa(0.) : ftoa(trChannel.ValueStartOrOffset.dVal), "StartValue");
	trChannelInfo.AddTextNode(trChannel.ChannelType.strVal!=STR_DATA_STORE_IMPLICIT? ftoa(0.) : ftoa(trChannel.ValueStepOrFactor.dVal), "StartWidth");
	trChannelInfo.Enable = ENABLE_READ_ONLY;
	trChannelInfo.SetAttribute(STR_ATTRIB_BRANCH, GETNBRANCH_OPEN);
	///---Sim 02-05-2010 QA81-15063 ROLL_BACK_MOVE_COL_INFO_OUT_OF_USER_TREE
	//trChannelInfo.Show = 0; ///---Sim 02-04-2010 QA81-15063 MOVE_IMP_FILE_INFO_OUT_FROM_COL_USER_INFO_TREE
	//bool bRet = set_user_info(col, strChannelInfo, trChannelInfo);
	bool bRet = fu_set_import_file_info(col, trChannelInfo, strChannelInfo, IMPORT_INFO_TO_USER_TREE);
	///---END QA81-15063 ROLL_BACK_MOVE_COL_INFO_OUT_OF_USER_TREE
	///---Sim 02-04-2010 QA81-15063 MOVE_IMP_FILE_INFO_OUT_FROM_COL_USER_INFO_TREE
	///---Sim 02-05-2010 QA81-15063 ROLL_BACK_MOVE_COL_INFO_OUT_OF_USER_TREE
	// roll back move column info out of user tree, as CP said
	//fu_set_import_file_name_info(col, m_strDatFileName);
//
	//trChannelInfo.RemoveChild("ImportFile");
	//trChannelInfo.Show = 1;
	//set_import_file_info(col, trChannelInfo, "DiademChannelInfo");
	///---END QA81-15063 ROLL_BACK_MOVE_COL_INFO_OUT_OF_USER_TREE
	///---END QA81-15063 MOVE_IMP_FILE_INFO_OUT_FROM_COL_USER_INFO_TREE
}

void DiademFile::getMissingValue(TreeNode& trChannel)
{
	m_bChannelHasMissing = trChannel.MissingValueFlag.strVal == STR_YES?  true : false;

	if(trChannel.MissingValue)
		m_dChannelMissing = trChannel.MissingValue.dVal;
	else
		m_dChannelMissing = m_trHeader.MissingValue.strVal == STR_NO_VALUE? DEFAULT_FILE_MISSING : m_trHeader.MissingValue.dVal;
}

void DiademFile::handelFactorAndOffset(TreeNode& trChannel)
{
	m_dOffset = trChannel.ValueStartorOffset? trChannel.ValueStartorOffset.dVal : 0.;
	m_dFactor = trChannel.ValueSteporFactor? trChannel.ValueSteporFactor.dVal : 1.;
}

/// Hong 04/16/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
//int DiademFile::handleColumnInfo(Worksheet& wks, Dataset& ds, TreeNode& trChannel, int nColIndex, int nChannelIndex)
int DiademFile::handleColumnInfo(Worksheet& wks, Dataset& ds, TreeNode& trChannel, int nColIndex, int nChannelIndex, bool bNeedOffsetDateData/* = true*/)
/// end FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
{
	ds.Attach(wks, nColIndex);
	
	if(!trChannel.DisplayFormat)
		return DIADEM_ERR_INVALID_FILE;
	
	Column col = wks.Columns(nColIndex);
	
	int nColFmt = col.GetFormat();
	
	if(trChannel.DisplayFormat.strVal == STR_TIME)
	{
		//block store and ASCII file will call impASC, it will auto handle col type,
		//if col tyoe is text or mixed, means read as string, no need convert
		if(nColFmt != OKCOLTYPE_TEXT_NUMERIC && nColFmt != OKCOLTYPE_TEXT)
			/// Hong 04/16/09 QA80-9828-P13 FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
			//handleDateData(col);
			handleDateData(col, bNeedOffsetDateData);
			/// end FIX_DIADEM_TIME_FMT_DATA_NOT_CORRECTLY_IMPORTED
	}
	else //set all other as numeric type
	{
		col.SetFormat(OKCOLTYPE_NUMERIC);
	}
	
	getMissingValue(trChannel);
	//if text format, no need to replaced
	if(m_bChannelHasMissing && col.GetFormat() == OKCOLTYPE_NUMERIC)
	{
		//should copy to vector, then get it again, otherwise missing value will be /0, not --
		ds.Replace(m_dChannelMissing, NANUM, MATREPL_TEST_EQUAL);
		vector vtemp(ds);
		ds= vtemp;
	}

	handelColumnParameter(col, trChannel, nChannelIndex, nColIndex);
	handelColumnUserInfo(col, trChannel);
	col.SetType(OKDATAOBJ_DESIGNATION_Y);
	
	return DIADEM_NO_ERROR;
}

///////////////////////////////////////////////////////////////////////////////////////////
//// testing codes
void testdiadem(int nFileDlg = 1)
{
	int ddcError = 0;
	string strFileName;
	
	if (nFileDlg)
		strFileName = GetOpenBox("[*.dat] *.dat");
	else
		strFileName = "d:\\text.DAT";
	
	Tree trFileNode;
	Tree trInfo;
	
	Worksheet wks = Project.ActiveLayer();
	
	DiademFile diademFile(strFileName);
	diademFile.Import(wks, trFileNode, trInfo);
}
