/*------------------------------------------------------------------------------*
 * File Name: minitab.c															*
 * Creation: GD 7/10/2001														*
 * Purpose: Origin C file														*
 * Copyright (c) OriginLab Corp. 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007	*
 * All Rights Reserved															*
 * 																				*
 * Modification Log:															*
 *	Cheney 7/5/2006 REWRITE_FILE												*
 *  Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8									*
 *  Hong 11/15/06 FIX_CHECK_MINITAB_MISSING_VALUE								*
 *	Cheney 2006-12-29 ONE_MPJ_PUT_INTO_ONE_BOOK									*
 *	Hong 05/26/08 v8.0870 FIX_LONG_TEXT_TRUNCATED_BY_TEXT_COL_LENGTTH_LIMITATION*
 *	Sim 02-04-2010 QA81-15063 MOVE_IMP_FILE_INFO_OUT_FROM_COL_USER_INFO_TREE	*
 *	Sim 02-05-2010 QA81-15063 ROLL_BACK_MOVE_COL_INFO_OUT_OF_USER_TREE			*
 *------------------------------------------------------------------------------*/
 
////////////////////////////////////////////////////////////////////////////////////
// you must include this header file for all Origin built-in functions and classes
#include <origin.h>
//
////////////////////////////////////////////////////////////////////////////////////


////////////////////////////////////////////////////////////////////////////////////
// start your functions here
#include "minitab.h"
#include "oDcFL.h"
#include "fu_utils.h" /// Hong 05/30/07 ADD_DATARANGE_FOR_ORGANIZER
// 1,2,4,8, default is 8
#pragma pack(push,1)

  //First part of file - 80 bytes
  typedef struct tagHeader1 {
 	int		var1;		// Unknown	1
 	int		var2;		// Unknown	36
 	int		var3;		// Unknown	12
 	int		var4;		// Unknown	0
 	char	ID[16]; 	// "MTB13   WIN     " That 'WIN' might be 'MAC' on MAC systems - not tested
 	int		var5;		// Unknown	Large and varies : 9.27731E8 to 9.71718E8 for example
 	int		var6;		// Unknown	5
 	int		var7;		// Unknown	32
 	int		var8;		// Unknown	12
 	int		NCols;		// Number of Columns - last column with data, counts those without data
 	int		NRows;		// Number of Rows - last row with data
 	int		Mult;		// Product of NCols and NRows
 	int		var12;		// Unknown	0
 	int		var13;		// Unknown	0
 	int		var14;		// Unknown	6
 	int		var15;		// Unknown	varies : 23 to 29 for example
 	int		var16;		// Unknown	12
 } Header1;
  

 //No real information on this data - 60 bytes
 typedef struct tagHeader2 {
 	int		var1;		// Unknown	7
 	int		var2;		// Unknown	Mostly 115, sometimes 111 or 125
 	int		var3;		// Unknown	12
 	int		var4;		// Unknown	1
 	int		var5;		// Unknown	0 (saw one instance of 1)
 	int		var6;		// Unknown	0
 	int		var7;		// Unknown	Mostly 1, also 4, 5, 7, 8, 9, 12
 	int		var8;		// Unknown	Mostly 0, also 1, 3, 17, 18, 30
 	int		var9;		// Unknown	1
 	int		var10;		// Unknown	Mostly 1, also 18, 23, 77
 	int		var11;		// Unknown	8
 	int		var12;		// Unknown	0
 	int		var13;		// Unknown	2
 	int		var14;		// Unknown	10
 	int		var15;		// Unknown	10
 } Header2;
 
 
/*
 * Some real guesses made here due to the odd length - 37
 * MUST use struct, can't use typedef struct
 * Two files were found that did not have a var9
 * I included code to 'back-up' (4 bytes) and test for a '3'
 * which should be the start of column data.
 * For the limited number of version 12 files I have,
 * there have been no var9 and no var8.
 * I included code to 'back-up' (12 bytes) and test for a '3'
 */
 struct Header3 {
 	int		var1;		// Unknown	1
 	int		var2;		// Unknown	42
 	char	var3;		// Unknown	0
 	int		var4;		// Unknown	1
 	int		var5;		// Unknown	1
 	int		var6;		// Unknown	1
 	int		var7;		// Unknown	1
 	double	var8;		// Unknown	8 - possibly a default width (Row headers?)
 	int		var9;		// Unknown	1
 };

/*
 * START OF COLUMN DATA
 * That should bring us to the data which is stored in column order
 * There are three formats : Text, Numeric and Date/Time
 * which have some differences in header length
 * I split the header into three parts: preamble, amble (ha, ha) and postamble (ha, ha, ha)
 * The preamble length is 'always' 60
 */

/*
 * The preamble - 60 bytes
 * When format type is Text and SortOrder is User Sort
 * then there are two blocks of data. The code always reads the second block.
 */
 typedef struct tagPre {
	int 	var1;		// Unknown	3
	int 	var2;		// Unknown	varies by format type
	int 	var3;		// Unknown	varies by format type
	int 	ColNum;		// The Column number (indexed from 1)
	int 	var5;		// Unknown	1 for Numeric and Date/Time, 2 for Text
	int		var6;		// Unknown	8 for Numeric and Date/Time, 14 for Text
	int 	Rows1;		// Number of rows - first copy (?)
	int 	var8;		// Unknown	28
	int 	Rows2;		// Number of rows - second copy (sometimes)
	int 	var10;		// Unknown	0
	int 	var11;		// Unknown	8 for Numeric and Date/Time, 0 for Text
	int 	var12;		// Unknown	0
	int 	var13;		// Unknown	8 for Numeric and Date/Time, 0 for Text
	int 	var14;		// Unknown	0
	int 	SortOrder;	// 0 is 'Alpha Sort', 1 is 'Occurrence Sort', 2 is 'User Sort'
 } Pre;

/*
 * The amble is two strings (integer length, followed by characters) and an integer
 * Each of the strings can be empty - indicated by length zero
 */

/*
 * The postamble length appears to be Format and version specific
 * Version 13
 *	Numeric - 43 bytes
 *	Date/Time - 47 bytes
 *	Text - 35 bytes
 * Version 12
 *	Numeric - 34 bytes
 *	Date/Time - ??? bytes
 *	Text - 26 bytes
 */


 //The postamble (v13), Numeric - 43 bytes
 struct PostN {
	int 	var1;		// Unknown 39
	USHORT 	var2;		// Unknown 8
	USHORT 	var3;		// Unknown 8
	USHORT 	var4;		// Unknown 8
	int 	var5;		// Unknown 0,1
	int 	var6;		// Unknown 1
	int 	var7;		// Unknown 0,1
	int 	var8;		// Unknown 1
	int 	var9;		// Unknown 0
	USHORT 	var10;		// Unknown 5
	USHORT 	var11;		// Unknown 1
	double 	Width;		// Column Width
	char 	Hidden;		// Column Hidden flag - 0 is show, 1 is hidden
 };


 //The postamble (v13), Date/Time - 47 bytes
 struct PostD {
	int		var1;		// Unknown 39
	USHORT	var2;		// Unknown
	USHORT	var3;		// Unknown
	USHORT	var4;		// Unknown
	int 	var5;		// Unknown 0
	int 	var6;		// Unknown
	int 	var7;		// Unknown 0
	int 	var8;		// Unknown
	char 	var9[4];	// Unknown - varies widely
	int 	var10;		// Unknown - very large or zero
	USHORT 	var11;		// Unknown
	USHORT 	var12;		// Unknown
	double 	Width;		// Column Width
	char 	Hidden;		// Column Hidden flag - 0 is show, 1 is hidden
 };


 //The postamble (v13), Text - 35 bytes
 struct PostT {
	int 	var1;		// Unknown 31
	USHORT 	var2;		// Unknown 8, 23
	USHORT 	NumChars;	// The maximum number of characters - seems to be used by every row
	USHORT 	var4;		// Unknown 8, 23
	int 	var5;		// Unknown 0,1
	int 	var6;		// Unknown 0,1
	int 	var7;		// Unknown 0,1
	int 	var8;		// Unknown 0,1
	double 	Width;		// Column Width
	char 	Hidden;		// Column Hidden flag - 0 is show, 1 is hidden
 };

 //The postamble (v12) Numeric - 34 bytes
 struct PostN12 {
	USHORT 	var1;		// Unknown 1
	USHORT 	var2;		// Unknown 2
	USHORT 	var3;		// Unknown 3
	USHORT 	var4;		// Unknown 4
	USHORT 	var5;		// Unknown 5
	USHORT 	var6;		// Unknown 6
	USHORT 	var7;		// Unknown 7
	USHORT 	var8;		// Unknown 8
	USHORT 	var9;		// Unknown 9
	USHORT 	var10;		// Unknown 10
	USHORT 	var11;		// Unknown 11
	USHORT 	var12;		// Unknown 12
	USHORT 	var13;		// Unknown 13
	USHORT 	var14;		// Unknown 14
	USHORT 	var15;		// Unknown 15
	USHORT 	var16;		// Unknown 16
	USHORT 	var17;		// Unknown 17
 };

 //The postamble (v12), Date/Time - 38? bytes
 struct PostD12 {
	int 	var1;		// Unknown1
	int 	var2;		// Unknown2
	int 	var3;		// Unknown3
	int 	var4;		// Unknown4
	int 	var5;		// Unknown5
	int 	var6;		// Unknown6
	int 	var7;		// Unknown7
	int 	var8;		// Unknown8
	int 	var9;		// Unknown9
	USHORT 	var10;		// Unknown10
 };	

 //The postamble (v12), Text - 26 bytes
 struct PostT12 {
	int 	var1;		// Unknown 31
	USHORT 	var2;		// Unknown 8, 23
	USHORT 	NumChars;	// The maximum number of characters - seems to be used by every row
	USHORT 	var4;		// Unknown 8, 23
	int 	var5;		// Unknown 0,1
	int 	var6;		// Unknown 0,1
	int 	var7;		// Unknown 0,1
	int 	var8;		// Unknown 0,1
 };

/*
 * Read the data here.
 * Numeric and Date/Time are both doubles.
 * The Date system used by MiniTab appears to be based on 12/30/1899.
 * so we must add 2451908 to date values.
 * Did not search for flags indicating Date or Time.
 * Release notes will warn that Dates around 1899, 1900 are probably Times.
 * For more columns, go back to START OF COLUMN DATA
 */

#pragma pack(pop)


#define		MINIMISSING				1.23456E30
#define		MINIMISSING2			1.234559E30 //Hong 11/15/06 FIX_CHECK_MINITAB_MISSING_VALUE
#define		DATEOFFSET				2415018
#define		SORTORDER_USER			2
#define 	BUFSIZE					256
#define		HEADER1SIZE				80
#define		HEADER2SIZE				60
#define		HEADER3SIZE				37
#define		PRESIZE					60
#define		VERSION12				12
#define		VERSION13				13
#define		CHKHEADER3VAR9			3
#define		CHKHEADER3VAR8AVAR9		3
#define		CHKVALIDCOLDATA			3
#define		TYPEUNKNOWNINFO			26
#define		CHKLENGTH				3


enum{
	DATATYPE_UNKNOWN	=	 0,
	DATATYPE_NUMERIC,	
	DATATYPE_DATE,			
	DATATYPE_TEXT
};

///---Sim 11-07-2006 REMOVE_TO_FILE_HEADER
/*
//Error Message
enum{
	FAILED_TO_READ_FILE_TO_TREE 			= 	-20,
	FAILED_TO_GET_DATA_FROM_STREAM,
	FAILED_TO_GET_SUBVEC_FROM_BYTE_VECTOR,
	COLUMN_SYNCHRONIZATION_FAILURE_V12,			
	COLUMN_SYNCHRONIZATION_FAILURE,					
	UNKNOWN_DATA_TYPE,			
	FILE_TOO_SMALL,				
	UNSUPPORT_VERSION,
	FAILED_TO_GET_FILE_HEADER,
	FAILED_TO_GET_DATA_HEADER,
	FAILED_TO_GET_DATA,
	FAILED_TO_GET_DATA_PREAMBLE,
	FAILED_TO_GET_DATA_POSTAMBLE,
	FAILED_TO_GET_LENGTHAND_SET_INFO,
};
*/
///---END REMOVE_TO_FILE_HEADER


//main
/// Hong 05/30/07 ADD_DATARANGE_FOR_ORGANIZER
//int readminitab(LPCSTR lpcszFilename, Worksheet& wks, TreeNode& trInfo, int nC1)
int readminitab(LPCSTR lpcszFilename, Worksheet& wks, TreeNode& trInfo, int nC1, DataRange& drImported) // = 0, = NULL
/// end ADD_DATARANGE_FOR_ORGANIZER
{
	Tree 		trFile;
	TreeNode 	trRoot = trFile.AddNode("RootStorage");
	
	if(cmpdoc_read_struct(lpcszFilename, &trRoot) != ODCFL_SUCCESS)
	{
		return MNTB_ERR_FAILED_TO_READ_FILE_TO_TREE;
	}
	
	int nWksNum 	= 	1;
	int nColsSum 	= 	0;
	/// Hong 11/7/06 FIX_NO_ERROR_MESSAGE_FOR_V14
	TreeNode trWks = trRoot.FindNodeByAttribute(STR_LABEL_ATTRIB, "Worksheet");
	if(!trWks.IsValid())
		return MNTB_ERR_UNSUPPORT_VERSION; ///---Sim 11-07-2006 HANDLE_ERR_MESSAGE_USE_DLL
	/// end FIX_NO_ERROR_MESSAGE_FOR_V14
		
	/// Hong 05/30/07 ADD_DATARANGE_FOR_ORGANIZER
	//int nRet = _read_each_stream(lpcszFilename, trRoot, wks, trInfo, nC1, nWksNum, nColsSum);
	int nRet = _read_each_stream(lpcszFilename, trRoot, wks, trInfo, nC1, nWksNum, nColsSum, drImported);
	/// end ADD_DATARANGE_FOR_ORGANIZER
	if(nRet < 0)
	{
		return nRet;
	}
	
	return nColsSum;
}

/// Hong 05/30/07 ADD_DATARANGE_FOR_ORGANIZER
//static int _read_each_stream(LPCSTR lpcszFilename, TreeNode &trStorage, Worksheet& wks, TreeNode& trInfo, int nC1, int& nWksNum, int& nColsSum)
static int _read_each_stream(LPCSTR lpcszFilename, TreeNode &trStorage, Worksheet& wks, TreeNode& trInfo, int nC1, int& nWksNum, int& nColsSum, DataRange& drImported)
/// end ADD_DATARANGE_FOR_ORGANIZER
{		
	int nRet = 0;
	foreach(TreeNode trNode in trStorage.Children)
	{
		if (trNode.tagName == ODCFLTREE_NODENAME_STREAM)
		{
			string strLabel;
			trNode.GetAttribute(STR_LABEL_ATTRIB, strLabel);
			
			if(strLabel != "Worksheet")
			{
				continue;
			}
			
			vector<byte> vb;
			if ( cmpdoc_read_stream(&vb, &trNode) != ODCFL_SUCCESS )
			{
				return MNTB_ERR_FAILED_TO_GET_DATA_FROM_STREAM;
			}
			
			Worksheet wksActive; // for reading more than one wks
			if(nWksNum != 1) 
			{
				///Cheney 2006-12-29 ONE_MPJ_PUT_INTO_ONE_BOOK
				//wksActive.Create();
				WorksheetPage wp;
				wks.GetParent(wp);
				/// Hong 02/12/07 ONLY_ADD_LAYER_IF_NOT_ENOUGH
				//wp.AddLayer();
				if( wp.Layers.Count()< nWksNum )
					wp.AddLayer();
				/// end ONLY_ADD_LAYER_IF_NOT_ENOUGH
				wksActive = wp.Layers(nWksNum - 1);
				///end ONE_MPJ_PUT_INTO_ONE_BOOK
			}
			else
			{
				wksActive = wks;
			}
			nWksNum++;
			
			int nRet = _get_wks_from_byte_vec(lpcszFilename, vb, wksActive, trInfo, nC1);	
			if(nRet < 0)
			{
				return nRet;
			}
			nColsSum += nRet;
			/// Hong 05/30/07 ADD_DATARANGE_FOR_ORGANIZER
			if ( drImported )
				drImported.Add(wksActive, nC1, STR_RANGE_STR, wksActive.GetNumCols() - 1);
			/// end ADD_DATARANGE_FOR_ORGANIZER
		}
		
		else if ( trNode.tagName == ODCFLTREE_NODENAME_SUBSTORAGE )
		{
			/// Hong 05/30/07 ADD_DATARANGE_FOR_ORGANIZER
			//nRet = _read_each_stream(lpcszFilename, trNode, wks, trInfo, nC1, nWksNum, nColsSum);
			nRet = _read_each_stream(lpcszFilename, trNode, wks, trInfo, nC1, nWksNum, nColsSum, drImported);
			/// end ADD_DATARANGE_FOR_ORGANIZER
			if(nRet < 0)
			{
				break;
			}
		}
	}

	return nRet;
}


#define GET_SUBVECTOR_AND_MEMCPY(_vSub, _vSrc, _nEnd, _cpyDest, _nLength ) \
			if(!_get_sub_vector(_vSub, _vSrc, _nLength, _nEnd)) return MNTB_ERR_FAILED_TO_GET_SUBVEC_FROM_BYTE_VECTOR; \
			memcpy(&_cpyDest, _vSub, _nLength);

#define GET_SUBVECTOR_AND_PUT_INFO_TO_BUF(_vSub, _vSrc, _nEnd, _chBuffer, _nLength ) \
			if(!_get_sub_vector(_vSub, _vSrc, _nLength, _nEnd)) return MNTB_ERR_FAILED_TO_GET_SUBVEC_FROM_BYTE_VECTOR; \	
			_put_info_to_buffer(_chBuffer, _vSub, _nLength);


static int _get_wks_from_byte_vec(LPCSTR lpcszFilename, vector<byte>& vb, Worksheet& wks, TreeNode& trInfo, int nC1)
{	
	if( vb.GetSize() < HEADER1SIZE + HEADER2SIZE + HEADER3SIZE + PRESIZE)
	{
		return MNTB_ERR_FILE_TOO_SMALL;
	}
	
	int nEnd = -1;
	int nNumCols;
	int nVersion;
	if(_read_file_header(vb, nEnd, wks, nNumCols, nVersion) < 0)
	{
		return MNTB_ERR_FAILED_TO_GET_FILE_HEADER;
	}
	
	//add columns if need
	int nColEnd = nNumCols + nC1; 
	//for(int nLoop = wks.GetNumCols(); nLoop < nColEnd; nLoop = wks.GetNumCols() ) 
	//{
		//wks.AddCol();
	//}
	if(nColEnd > wks.GetNumCols())
		wks.SetSize(-1, nColEnd);
	
	if(_read_data_header(vb, nEnd, wks, nVersion) < 0)
	{
		return MNTB_ERR_FAILED_TO_GET_DATA_HEADER;
	}
	
	// Use this to replace MiniTab missing values with Origin missing values
	LT_set_var("MNTBMISSING", MINIMISSING);
	
	if(_read_data(lpcszFilename, vb, nEnd, wks, nNumCols, nC1, nVersion) < 0)
	{
		return MNTB_ERR_FAILED_TO_GET_DATA;
	}
	
	return nNumCols;
}


//read file header
static int _read_file_header(vector<byte>& vb, int& nEnd, Worksheet& wks, int& nNumCols, int& nVersion)
{
	// Read Header1 and determine version
	// I will let pass versions greater than 13 and attempt to read them as v13 (with warning)
	// I will let pass versions less than 12 and attempt to read them as v12 (with warning)
	Header1	Hdr1;	// size is 80
	vector<byte> vbSubTemp;
	GET_SUBVECTOR_AND_MEMCPY(vbSubTemp, vb, nEnd, Hdr1, sizeof(Hdr1))
	
	char chVersion[3];	// file version : 12 and 13 supported
	chVersion[0] = Hdr1.ID[3];
	chVersion[1] = Hdr1.ID[4];
	chVersion[2] = '\x00';
	nVersion = atoi(chVersion);
	
	nNumCols = Hdr1.NCols;
	return 0;
}


//read data header
static int _read_data_header(vector<byte>& vb, int& nEnd, Worksheet& wks, int nVersion)
{
	// Setup %Z in LabTalk - which will contain header text information
	// Strings in Minitab are an integer containing Length followed by Length characters
	char chBuffer[BUFSIZE];
	chBuffer[0] = 0;
	LT_set_str("%Z", chBuffer);
	
	// Read and append Name field (to %Z)
	int nLength;
	vector<byte> vbSubTemp;
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	//if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, nLength, true, true) < 0)
	if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, wks, nLength, true, true) < 0)
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
	{
		return MNTB_ERR_FAILED_TO_GET_LENGTHAND_SET_INFO;
	}
	
	// Read and append User Name field (to %Z)
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	//if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, nLength, true, false, true) < 0)
	if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, wks, nLength, true, false, true) < 0)
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
	{
		return MNTB_ERR_FAILED_TO_GET_LENGTHAND_SET_INFO;
	}
	
	// Read and append Date (to %Z) - could be any text actually
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	//if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, nLength, true, false, true) < 0)
	if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, wks, nLength, true, false, true) < 0)
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
	{
		return MNTB_ERR_FAILED_TO_GET_LENGTHAND_SET_INFO;
	}
	
	// Read and append Notes (to %Z)
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	//if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, nLength, true, false, true) < 0)
	if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, wks, nLength, true, false, true) < 0)
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
	{
		return MNTB_ERR_FAILED_TO_GET_LENGTHAND_SET_INFO;
	}
	
	// Read Header2 - I don't know what any of this means
	Header2	Hdr2;		// size is 60
	GET_SUBVECTOR_AND_MEMCPY(vbSubTemp, vb, nEnd, Hdr2, sizeof(Hdr2))

	// Read Data Font - the font for Data display - Don't use, but we could
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	//if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, nLength, true) < 0)
	if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, wks, nLength, true) < 0)
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
	{
		return MNTB_ERR_FAILED_TO_GET_LENGTHAND_SET_INFO;
	}
	
	// Read Label Font - the font for Label Display - Don't use, and we could not anyway
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	//if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, nLength, true) < 0)
	if(_get_length_and_set_info(vbSubTemp, vb, nEnd, chBuffer, wks, nLength, true) < 0)
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
	{
		return MNTB_ERR_FAILED_TO_GET_LENGTHAND_SET_INFO;
	}
	
	// Read Header3 - I don't know what any of this means, either
	Header3	Hdr3;		// size is 37
	GET_SUBVECTOR_AND_MEMCPY(vbSubTemp, vb, nEnd, Hdr3, sizeof(Hdr3))
		
	// Check for valid column data
	// This corrects a problem with a few version 13 files that are missing Hdr3.var9
	if(nVersion >= VERSION13 && Hdr3.var9 == CHKHEADER3VAR9)
	{
		nEnd -= sizeof(int);
	}
	
	// This corrects a difference for version 12 files that do not have Hdr3.var8 and Hdr3.var9
	if(nVersion <= VERSION12)
	{
		nEnd -= 3 * sizeof(int);
		// Now do our test for valid column data. This integer should be 3
		int nTemp;
		GET_SUBVECTOR_AND_MEMCPY(vbSubTemp, vb, nEnd, nTemp, sizeof(nTemp))
		
		if(nTemp != CHKHEADER3VAR8AVAR9)
		{
			return MNTB_ERR_COLUMN_SYNCHRONIZATION_FAILURE_V12;
		}
	}
	
	return 0;
}


#define GET_COLUMN_WIDTH(_vSub, _vSrc, _nEnd, _stVar1, _stVar2, _pWidth, _nVersion) \
		if(nVersion >= VERSION13) { \ 
			GET_SUBVECTOR_AND_MEMCPY(_vSub, _vSrc, _nEnd, _stVar1, sizeof(_stVar1)); \
			*_pWidth = _stVar1.Width; } \
		else GET_SUBVECTOR_AND_MEMCPY(_vSub, _vSrc, _nEnd, _stVar2, sizeof(_stVar2)); 

#define GET_COLUMN_STRSIZE(_vSub, _vSrc, _nEnd, _stVar1, _stVar2, _pStrSize, _nVersion) \
		if(nVersion >= VERSION13)  {*_pStrSize = _stVar1.NumChars;} \
		else {*_pStrSize = _stVar2.NumChars;}
				
//read data
static int _read_data(LPCSTR lpcszFilename, vector<byte>& vb, int& nEnd, Worksheet& wks, int nNumCols, int nC1, int nVersion)
{
	// For each column
	int nMaxRows = 0;
	for(int nLoop = 1; nLoop <= nNumCols; nLoop++)
	{
		vector<byte> vbSubTemp;
		int nSortOrder;
		int nDataType;
		int nThisCol;
		int nLength;
		int nRowNum;
		if(_read_data_preamble(vbSubTemp, vb, nEnd, wks, nRowNum, nSortOrder, nDataType, nC1, nThisCol, nLength) < 0)
		{
			return MNTB_ERR_FAILED_TO_GET_DATA_PREAMBLE;
		}
		
		// Read the Postamble based on the DataType - the actual data is read here
		int nStatus = 0;
		PostN NumericHdr;		// size is 43
		char chBuffer[BUFSIZE];
		int nWidth;
		int *pWidth = &nWidth;
		int nStrSize;
		int *pStrSize = &nStrSize;
		switch(nDataType)
		{
		case DATATYPE_UNKNOWN:
			if(nVersion == VERSION12 || nVersion == VERSION13)
			{
				if(nVersion == VERSION12)
				{
					nLength = TYPEUNKNOWNINFO;
					GET_SUBVECTOR_AND_PUT_INFO_TO_BUF(vbSubTemp, vb, nEnd, chBuffer, nLength )
				}
				else
				{
					GET_SUBVECTOR_AND_MEMCPY(vbSubTemp, vb, nEnd, NumericHdr, sizeof(NumericHdr))
				}
			}
			else
			{
				nStatus = MNTB_ERR_UNSUPPORT_VERSION;
			}
			break;
			
		case DATATYPE_NUMERIC:
			PostN12	Numeric12Hdr;	// size is 34
			GET_COLUMN_WIDTH(vbSubTemp, vb, nEnd, NumericHdr, Numeric12Hdr, pWidth, nVersion)
			if(_read_data_postamble(vbSubTemp, vb, nEnd, wks, nVersion, nRowNum, nThisCol, nWidth, nStrSize, DATATYPE_DATE) < 0)
			{
				return 	MNTB_ERR_FAILED_TO_GET_DATA_POSTAMBLE;
			}
			
			_read_numeric_data(vbSubTemp, wks, nThisCol, nRowNum);
			break;
			
		case DATATYPE_DATE:
			PostD DateHdr;	// size is 47
			PostD12	Date12Hdr;	// size is 38?
			GET_COLUMN_WIDTH(vbSubTemp, vb, nEnd, DateHdr, Date12Hdr, pWidth, nVersion)
			if(_read_data_postamble(vbSubTemp, vb, nEnd, wks, nVersion, nRowNum, nThisCol, nWidth, nStrSize ,DATATYPE_DATE) < 0)
			{
				return 	MNTB_ERR_FAILED_TO_GET_DATA_POSTAMBLE;
			}
			
			_read_date_data(vbSubTemp, wks, nThisCol, nRowNum);
			break;
	
		case DATATYPE_TEXT:
			PostT TextHdr;		// size is 35
			PostT12 Text12Hdr;	// size is 26
			GET_COLUMN_WIDTH(vbSubTemp, vb, nEnd, TextHdr, Text12Hdr, pWidth, nVersion)
			GET_COLUMN_STRSIZE(vbSubTemp, vb, nEnd, TextHdr, Text12Hdr, pStrSize, nVersion)
			if(_read_data_postamble(vbSubTemp, vb, nEnd, wks, nVersion, nRowNum, nThisCol, nWidth, nStrSize) < 0)
			{
				return 	MNTB_ERR_FAILED_TO_GET_DATA_POSTAMBLE;
			}
			
			nStatus = _read_text_data(vb, nEnd, wks, nThisCol, nRowNum, nSortOrder, nStrSize, nVersion);
			break;

		default:
			nStatus = MNTB_ERR_UNKNOWN_DATA_TYPE;
		}	// end of switch based on DATATYPE
		
		if(nStatus < 0)
		{
			return nStatus;
		}
		nMaxRows = nMaxRows < nRowNum? nRowNum : nMaxRows;
		
		///---Sim 02-04-2010 QA81-15063 MOVE_IMP_FILE_INFO_OUT_FROM_COL_USER_INFO_TREE
		//string strColumnInfo = "ColumnInfo";
		//Tree trColumnInfo;
		//trColumnInfo.AddTextNode(lpcszFilename, "ImportFile");
		//trColumnInfo.Enable = ENABLE_READ_ONLY;
		//trColumnInfo.SetAttribute(STR_ATTRIB_BRANCH, GETNBRANCH_OPEN);
		//bool bRet = set_user_info(wks.Columns(nThisCol), strColumnInfo, trColumnInfo);	
		fu_set_import_file_name_info(wks.Columns(nThisCol), lpcszFilename, IMPORT_INFO_TO_USER_TREE);
		///---Sim 02-05-2010 QA81-15063 ROLL_BACK_MOVE_COL_INFO_OUT_OF_USER_TREE
		// roll back move column info out of user tree, as CP said
		//fu_set_import_file_name_info(wks.Columns(nThisCol), lpcszFilename);
		///---END QA81-15063 ROLL_BACK_MOVE_COL_INFO_OUT_OF_USER_TREE
		///---END QA81-15063 MOVE_IMP_FILE_INFO_OUT_FROM_COL_USER_INFO_TREE
	}	// end of 'for each column'
	
	#if _OC_VER < 0x0800
	LT_execute("wks.labels();");
	#endif //_OC_VER < 0x0800
	///Cheney 2006-12-29 ONE_MPJ_PUT_INTO_ONE_BOOK
	//wks.SetSize(-1, nNumCols);
	///end ONE_MPJ_PUT_INTO_ONE_BOOK
	return 0;
}


static int _read_data_preamble(vector<byte>& vbSub, vector<byte>& vbSrc, int& nEnd, Worksheet& wks,
				 int& nRowNum, int& nSortOrder, int& nDataType, int nC1, int& nThisCol, int& nLength)
{
	// Read the Preamble - Column Number, Number of Rows and SortOrder flag
	Pre	DataPreamble;	// size is 60 
	GET_SUBVECTOR_AND_MEMCPY(vbSub, vbSrc, nEnd, DataPreamble, sizeof(DataPreamble))
	
	// This is my test for valid column data - see Check for valid column data
	if(DataPreamble.var1 != CHKVALIDCOLDATA)
	{
		return MNTB_ERR_COLUMN_SYNCHRONIZATION_FAILURE;
	}
	
	// Read the Amble - contains Column Label, Column Description and DataType
	
	// Column Label
	nThisCol = DataPreamble.ColNum + nC1 - 1;
	char chBuffer[BUFSIZE];
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	//if(_get_length_and_set_info(vbSub, vbSrc, nEnd, chBuffer, nLength, true, false, false, true, nThisCol + 1) < 0)
	if(_get_length_and_set_info(vbSub, vbSrc, nEnd, chBuffer, wks, nLength, true, false, false, true, nThisCol + 1) < 0)
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
	{
		return MNTB_ERR_FAILED_TO_GET_LENGTHAND_SET_INFO;
	}
	
	// Column Description
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	//if(_get_length_and_set_info(vbSub, vbSrc, nEnd, chBuffer, nLength, true) < 0)
	if(_get_length_and_set_info(vbSub, vbSrc, nEnd, chBuffer, wks, nLength, true) < 0)
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
	{
		return MNTB_ERR_FAILED_TO_GET_LENGTHAND_SET_INFO;
	}
	
	// Column DataType
	GET_SUBVECTOR_AND_MEMCPY(vbSub, vbSrc, nEnd, nDataType, sizeof(nDataType))
	
	nRowNum = DataPreamble.Rows1;
	
	return 0;
}


static int _read_data_postamble(vector<byte>& vbSub, vector<byte>& vbSrc, int& nEnd, Worksheet& wks,
				 int nVersion, int nRowNum, int nThisCol, int nWidth, int nStrSize, int nType = DATATYPE_TEXT)
{
	if(nVersion >= VERSION13)
	{
		#if _OC_VER < 0x0800
		LT_set_var("mntbthiscol", nThisCol - 1);
		LT_set_var("mntbthiswidth", nWidth);
		if(nType != DATATYPE_TEXT)  //DATATYPE_NUMERIC or DATATYPE_DATE
		{
			LT_execute("wks.col$(mntbthiscol).width=mntbthiswidth;");
		}
		else
		{
			LT_set_var("mntbthislen", nStrSize + 1);
		}
		#endif //_OC_VER < 0x0800
	}
	
	if(nType != DATATYPE_TEXT)	//DATATYPE_NUMERIC or DATATYPE_DATE
	{
		if(!_get_sub_vector(vbSub, vbSrc, nRowNum * sizeof(double), nEnd))
		{
			return MNTB_ERR_FAILED_TO_GET_SUBVEC_FROM_BYTE_VECTOR;
		}
	}
	
	return 0;
}


//get vbSub from vbSrc, nLength is vbSub's size
//iEnd returns vbSub last element's position in vbSrc.
static BOOL _get_sub_vector(vector<byte>& vbSub, vector<byte>& vbSrc, int nLength, int& nEnd)
{
	int nBegin = nEnd + 1;
	nEnd = nEnd + nLength;
	
	if(vbSrc.GetSubVector(vbSub, nBegin, nEnd) < 0)
	{
		return FALSE;
	}
	
	return TRUE;
}	


static void _put_info_to_buffer(char* chBuffer, vector<byte>& vbSrc, int nLength)
{
	memcpy(chBuffer, vbSrc, nLength);
	chBuffer[nLength] = 0;
}


// Read Numeric data into the correct column of the worksheet
static void _read_numeric_data(vector<byte>& vbSrc, Worksheet& wks, int nTheColumn, int nNumRows)
{	
	// link to dataset, set column format to Numeric, set Dataset size
	Dataset	ds(wks.Columns(nTheColumn));
	ds.SetSize(nNumRows);
	wks.Columns(nTheColumn).SetFormat(OKCOLTYPE_NUMERIC);
	
	vector	vect(nNumRows);	
	// Read the data into Origin, replace MiniTab Missing with Origin Missing			
	memcpy(&vect, vbSrc, vbSrc.GetSize());	
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	#if _OC_VER < 0x0800
		ds = vect;
		LT_execute("wcol(mntbthiscol)=wcol(mntbthiscol)==MNTBMISSING?0/0:wcol(mntbthiscol);");
	#else // _OC_VER < 0x0800
		for(int ii=0; ii < nNumRows; ii++)
		{
			/// Hong 11/15/06 FIX_CHECK_MINITAB_MISSING_VALUE
			// mititab's miss value is not a value exactly eqaul to 1.23456E30,
			// but a little small than it, nearly 140737488355328
			//if( MINIMISSING == vect[ii] )			 							   
			if( MINIMISSING > vect[ii] && vect[ii] > MINIMISSING2 )			
			/// end FIX_CHECK_MINITAB_MISSING_VALUE
				vect[ii] = NANUM;
		}
		ds = vect;
	#endif // _OC_VER < 0x0800
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
}


// Read Date data into the correct column of the worksheet
static void _read_date_data(vector<byte>& vbSrc, Worksheet& wks, int nTheColumn, int nNumRows)
{	
	// link to dataset, set column format to Date, set Dataset size
	Dataset	ds(wks.Columns(nTheColumn));
	ds.SetSize(nNumRows);
	wks.Columns(nTheColumn).SetFormat(OKCOLTYPE_DATE);
	
	vector	vect(nNumRows);
	// Read the data into Origin, replace MiniTab Missing wit Origin Missing
	memcpy(&vect, vbSrc, vbSrc.GetSize());	
	/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
	#if _OC_VER < 0x0800
		ds = vect;
		LT_execute("wcol(mntbthiscol)=wcol(mntbthiscol)==MNTBMISSING?0/0:wcol(mntbthiscol);");
		
		// Adjust for MiniTab Date system
		LT_execute("wcol(mntbthiscol)+=mntbDATEOFFSET;");
	#else // _OC_VER < 0x0800
		for(int ii=0; ii < nNumRows; ii++)
			if( MINIMISSING == vect[ii] )
				vect[ii] = 0/0;
		vect += DATEOFFSET;
		ds = vect;
	#endif // _OC_VER < 0x0800
	/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
}


// Read Text data into the correct column of the worksheet
static int _read_text_data(vector<byte>& vbSrc, int& nEnd, Worksheet& wks, int nTheColumn, int nNumRows, int nTheSortOrder, int nTheStringSize, int nTheVersion)
{
	char	chBuffer[BUFSIZE];
	
	Dataset	ds(wks.Columns(nTheColumn));
	ds.SetSize(nNumRows);
	
	// link to dataset, set column format to Text, set dataset size
	/// Hong 05/26/08 v8.0870 FIX_LONG_TEXT_TRUNCATED_BY_TEXT_COL_LENGTTH_LIMITATION
	// CP said that OKCOLTYPE_TEXT is for short text string, only OKCOLTYPE_TEXT_NUMERIC can support long string up to 255
	//wks.Columns(nTheColumn).SetFormat(OKCOLTYPE_TEXT);
	wks.Columns(nTheColumn).SetFormat(OKCOLTYPE_TEXT_NUMERIC);
	/// end FIX_LONG_TEXT_TRUNCATED_BY_TEXT_COL_LENGTTH_LIMITATION
	
	if(nTheVersion >= VERSION13)
	{
		#if _OC_VER < 0x0800
		LT_execute("wks.col$(mntbthiscol).width=mntbthiswidth;");
		LT_execute("wks.col$(mntbthiscol).twidth=mntbthislen;");
		#endif //_OC_VER < 0x0800
	}
	
	// For each row
	int	nLength;
	vector<byte> vbSubTemp;
	for(int nLoop = 0; nLoop < nNumRows; nLoop++)
	{
		GET_SUBVECTOR_AND_MEMCPY(vbSubTemp, vbSrc, nEnd, nLength, sizeof(nLength));
		GET_SUBVECTOR_AND_PUT_INFO_TO_BUF(vbSubTemp, vbSrc, nEnd, chBuffer, nLength );
		wks.SetCell(nLoop, nTheColumn, chBuffer);
	}

	// for Text - check for SortOrder of User AND that the first integer is not 3
	/// Hong 10/20/06 FIX_MISS_CASE_OF_LAST_COLUMN
	//GET_SUBVECTOR_AND_MEMCPY(vbSubTemp, vbSrc, nEnd, nLength, sizeof(nLength));
	if(nEnd < vbSrc.GetSize() - 1)
		GET_SUBVECTOR_AND_MEMCPY(vbSubTemp, vbSrc, nEnd, nLength, sizeof(nLength));
	/// end FIX_MISS_CASE_OF_LAST_COLUMN
	nEnd -= sizeof(nLength);	// go back
	
	// Only Text data has this feature, just guessing when datablock is 'doubled
	if(SORTORDER_USER == nTheSortOrder && nLength != CHKLENGTH)
	{
		nEnd += (nTheVersion >= VERSION13) ? nNumRows * (sizeof(nLength) + nTheStringSize) 
						: nNumRows * (sizeof(nLength) + nTheStringSize);
	}
	
	return 0;
}

/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
//static int _get_length_and_set_info(vector<byte>& vbSub, vector<byte>& vbSrc, int& nEnd, char* chBuffer,
//						int nLength, bool bPutToBuf = true, bool bSetUserName = false, bool bSetApnName = false,
//						bool bSetColLabel = false, int nCol = 0)
static int _get_length_and_set_info(vector<byte>& vbSub, vector<byte>& vbSrc, int& nEnd, char* chBuffer, Worksheet& wks,
						int nLength, bool bPutToBuf = true, bool bSetUserName = false, bool bSetApnName = false,
						bool bSetColLabel = false, int nCol = 0)
/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
{
	GET_SUBVECTOR_AND_MEMCPY(vbSub, vbSrc, nEnd, nLength, sizeof(nLength));
	
	if(nLength) 
	{ 
		GET_SUBVECTOR_AND_PUT_INFO_TO_BUF(vbSub, vbSrc, nEnd, chBuffer, nLength ); 
		
		if(bPutToBuf)
		{
			LT_set_str("%N", chBuffer); 
		}
		if(bSetUserName)
		{
			///Cheney 2006-12-29 ONE_MPJ_PUT_INTO_ONE_BOOK
			//LT_execute("page.label$=%N;page.title=3;%Z=%Z (%N)");
			wks.SetName(chBuffer);
			///end ONE_MPJ_PUT_INTO_ONE_BOOK
		}
		if(bSetApnName)
		{
			LT_execute("%Z=%Z (%N);");
		}
		if(bSetColLabel)
		{
			/// Hong 11/15/06 CLEAN_NOT_USE_LT_IN_ORIGIN8
			#if _OC_VER < 0x0800
				LT_set_var("mntbthiscol", nCol);
				LT_execute("wks.col$(mntbthiscol).label$=%N;");
			#else
				//wks.Columns(nCol).SetLabel(chBuffer);
				wks.Columns(nCol - 1).SetLabel(chBuffer);// Hong 11/21/06 FIX_DIFF_OFFSET_BETWEEN_LT_AND_OC
			#endif
			/// end CLEAN_NOT_USE_LT_IN_ORIGIN8
		}
	}
	
	return 0;
}

