package src.com.quarch.deviceMOM;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;

import src.com.quarch.beCommandData.CmdStruct;
import src.com.quarch.beStream.StreamBufferStriped;
import src.com.quarch.beStream.StreamDataReadyListener;
import src.com.quarch.beStream.StreamEventListener;
import src.com.quarch.beStream.StreamFactory;
import src.com.quarch.beStream.StreamStatusCodes;
import src.com.quarch.beStreamData.StreamRawData;
import src.com.quarch.deviceInterface.DeviceListEntry;
import src.com.quarch.devices.BaseStreamDevicePPM;
import src.com.quarch.devices.BufferPage;
import src.com.quarch.devices.DeviceDataBuffer;
import src.com.quarch.devices.PagedBuffer;
import src.com.quarch.utils.DebugUtil;

public class MOM extends BaseStreamDevicePPM implements StreamEventListener {



	public MOM (DeviceListEntry dle) {
		myDle = dle;
		this.setHasREST( true );
		this.setHasSerial( false );
		this.setHasStream( true );			// mom supports streaming but only if a fixture is attached, need code to account for this
		this.setHasTelnet( false );
		this.setHasUDP( true );
		this.setHasUSB( true );

		basePPM.headerVersion = PPMHeaderVersion.V3;

		dle.deviceInfo.cmdBufferSize = 512;
		dle.deviceInfo.streamBufferSize = 512;

		streamStatusCode = StreamStatusCodes.STOPPED;
		streamBuffer = new StreamBufferStriped();
		//
		// MoM will provide this information directly from the attached fixture
		//
//		this.getDeviceOptions().put(CONF_OUT_MODE, null);
//
//		final long vMax = (int) Math.pow(2, 14);
//		final long cMax = (int) Math.pow(2, 24);
//		final long pMax = vMax * cMax / 1000;			// min of power is in uA so div by 1000. Also power maximums will be limited by device protection
//
//		setTheoreticalPPMMaximums( vMax, cMax, vMax, cMax, pMax, pMax, pMax + pMax );
	}


	@Override
	public void enableStreamReceive() {
		boolean startOkay = false;

		if( myDle.deviceInfo.openForStream() ) {
			try {
				myDle.deviceInfo.cmdLock.acquireUninterruptibly();
				streamBase = StreamFactory.getInstance().createNewStream(myDle);
				streamBase.streamEventInitiater.addListener(this);
				steamThread = new Thread( streamBase );
				streamState = StreamState.waitHeader;

				streamBuffer.clear();								// intention is to remove this for MoM
				pagedBuffer.clear();
				binaryStreamDataBuffer = null;


				streamHeader = null;
				steamThread.start();
				running = true;
				recordCount = 0;
				startOkay = true;
			}
			finally {
				if( !startOkay ) {
					myDle.deviceInfo.closeStream();
					myDle.deviceInfo.cmdLock.release();				// some problem starting the stream, release the cmd lock
				}
				else
					streamStatusCode = StreamStatusCodes.RUNNING;
			}
		}
	}


	@Override
	public void disableStreamReceive() {
//		streamRunning = false;
		myDle.deviceInfo.closeStream();

		if( streamStatusCode == StreamStatusCodes.RUNNING )
			streamStatusCode = StreamStatusCodes.STOPPED; 			// will be overwritten by stop
		// else accept whatever the non-running code happens to be

		streamBase.fireDataReadyEvent( streamStatusCode, streamBuffer.size() );

		streamBase.stop();
		streamBase = null;
		running = false;
	}


	@Override
	public boolean internalCommand( CmdStruct cmd ) {
		int maxReadStripes = 3072;			// 4096;			// 8192;
		String sCmd = cmd.command.replaceAll("(\\r|\\n)", "");

		if( sCmd.toLowerCase().equals("stream?"))
			cmdStreamQuery(cmd);
		else if( sCmd.toLowerCase().startsWith("stream text") )					// MoM need to fix/implement count of buffered entries for "stream?"
			cmdStreamText(cmd, maxReadStripes);
		else if( sCmd.toLowerCase().startsWith("stream bin") )
			cmdStreamBin(cmd, maxReadStripes);

		else if( modePowerCmd( sCmd, cmd ) ) {
		}

		else if( sCmd.toLowerCase().startsWith("stream mode header v3") ) {
			basePPM.headerVersion = PPMHeaderVersion.V3;
			cmd.response.add("OK");
			cmd.response.add(">");
			cmd.action = 333;
		}
		else if( sCmd.toLowerCase().startsWith("stream mode header v2") ) {		// we could down grade the provided header if necessary
			cmd.response.add("FAIL: Mode Not Supported");
			cmd.response.add(">");
			cmd.action = 333;
		}
		else if( sCmd.toLowerCase().startsWith("stream mode header v1") ) {		// we could down grade the provided header if necessary
			cmd.response.add("FAIL: Mode Not Supported");
			cmd.response.add(">");
			cmd.action = 333;
		}
		else if( sCmd.toLowerCase().startsWith("stream mode header?") ) {
			cmd.response.add( basePPM.headerVersion.getCode() );
			cmd.response.add(">");
			cmd.action = 333;
		}

		else if( sCmd.toLowerCase().startsWith("stream mode resample?") ) {				// need to migrate resample to xlc and ppm

			cmd.response.add(basePPM.getRequestedResample());
			cmd.response.add(">");
			cmd.action = 333;
		}
		else if( sCmd.toLowerCase().startsWith("stream mode resample") ) {
			if( running ) {
				cmd.response.add("FAIL: Cannot Change While Stream Running" );
			}
			else {
				String nStr = sCmd.trim();
				int pos = nStr.lastIndexOf(" ");
				if( pos < "stream mode resample".length() )
					nStr = "";
				else
					nStr = nStr.substring( pos + 1);

				if( nStr.toLowerCase().equals(BasePPM.requestedResampleOffString) ) {
					basePPM.setRequestedResample(BasePPM.requestedResampleOffString);
					cmd.response.add("OK");
				}
				else
					if( basePPM.setValidatedRequestedResample(nStr) )
						cmd.response.add("OK");
					else
						cmd.response.add("FAIL: Bad Value (" + nStr + ")" );
			}
			cmd.response.add(">");
			cmd.action = 333;
		}

		return cmd.action == 333;
	}


	private void cmdStreamQuery(CmdStruct cmd) {		// from HD
		{
			switch( streamStatusCode ) {
				case STOPPED:
					cmd.response.add("Stopped:");
					break;
				case RUNNING:
					cmd.response.add("Running");
					break;

				case StoppedUser:
					cmd.response.add("Stopped: User");
					break;
				case StoppedOverrun:
					cmd.response.add("Stopped: Overrun");
					break;
				case StoppedOverTemp:
					cmd.response.add("Stopped: Over Temperature");
					break;
				case StopppedFixtureDisconnect:
					cmd.response.add("Stopped: Fixture Disconnect");
					break;
				case StoppedCommsFailure:
					cmd.response.add("Stopped: Comms Failure");
					break;
				default:
					break;
			}

			if( streamBuffer != null )
				cmd.response.add("Stripes Buffered: " + Integer.toString(streamBuffer.size()) + " of " + Integer.toString(streamBuffer.getMaxRamStripes()));
			else
				cmd.response.add("Stripes Buffered: 0");

			cmd.response.add(">");
        	cmd.action = 333;				// handled internally
		}
	}


	public static int[] convertIntegers(List<int[]> dataList)		// from HD
	{
	    int[] ret = new int[dataList.size()];
	    for (int i=0; i < ret.length; i++)
	    {
	        ret[i] = dataList.get(i).length;
	    }
	    return ret;
	}


	//
	// little endian
	//
	private void intAtoByteA( int[] ints, byte[] bytes, int startIdx ) {		// from HD
		int i;

		for( i = 0; i < ints.length; ++i ) {
//			try {
				bytes[startIdx++] = (byte)(  ints[i] & 0xff);
				bytes[startIdx++] = (byte)( (ints[i] >> 8) & 0xff);
				bytes[startIdx++] = (byte)( (ints[i] >> 16) & 0xff);
				bytes[startIdx++] = (byte)( (ints[i] >> 24) & 0xff);
//			}
//			catch( ArrayIndexOutOfBoundsException e ) {
//				DebugUtil.debugErrorln("intAtoByteA Index Error " + Integer.toString(i));
//			}
		}
	}




	private final static int bihDataHeaderSize = 4;		// from HD

	/*
	 * maxReadStripes is ignored, whole buffers, full or otherwise are returned as the CmdStruct bytebuffer field
	 *
	 * header:
	 *		int[0] byte[0] = 5, version number of page/group
	 *
	 * (previously !!) header:
	 *		int[0] 	byte[0] length of header in int32s
	 *		int[1]	elements (ints) per stripe
	 *		int[2] 	number of stripes
	 *		int[3]	Record number of first stripe in this packet
	 *
	 */
	private void cmdStreamBin(CmdStruct cmd, int maxReadStripes) {		// from HD

		try {
		final BufferPage page = pagedBuffer.getFullBufferPage();

		if( page != null ) {
			page.getData().rewind();
			if( page.getData().getInt() != 5)
				page.getData().rewind();

			page.getData().rewind();

			//cmd.directResponseBuffer = page.getData();
			cmd.cmBuffer = page;
		}
		else {
			BufferPage empty = pagedBuffer.getEmptyBufferPage();
			
			empty.getData().putInt( 5 );		// header version 5
			empty.getData().putInt( 8 );		// # bytes in this buffer when passed down the line
			empty.getData().limit(8);
			empty.getData().rewind();
			
			cmd.cmBuffer = empty;
			
		}
		
		cmd.action = 333;				// handled internally
		}
		catch( Exception e ) {
			e.printStackTrace();
		}
//		int maxStripes = -1;
//		maxStripes = maxReadStripes;
//		byte[] dataArray = new byte[ (bihDataHeaderSize + (maxReadStripes * (StreamBufferStriped.maxStripeData + 1) )) * sizeOfInt ];		// header + max stripe bytes including trigger
//		int dataArrayIdx = (bihDataHeaderSize * sizeOfInt);
//
//		int[] firstRecord = new int[4];
//
//		firstRecord[0] = bihDataHeaderSize;						// length of first record
//
//		// need a header of some kind as first record?
//		if( (maxStripes > 0) && (maxStripes <= maxReadStripes) ) {
//			while( maxStripes > 0 ) {
//				StreamRawData rawData =  streamBuffer.getOldestStripe();		//streamBuffer.getSimulatedStripe( 500, 6 );					// 				// need to frig stripe for mom debugging!
//				if( rawData == null ) {
//					/// cmd.response.add( "eof" );
//					break;
//				}
//				if( firstRecord[2]++ == 0) 						// stripe count
//					firstRecord[3] = rawData.getRecNo();
//
//				int[] raw = rawData.asArrayMinusRecNo();
//
//				firstRecord[1] = raw.length;					// elements per stripe
//				intAtoByteA( raw, dataArray, dataArrayIdx );
//				dataArrayIdx = dataArrayIdx + (raw.length * sizeOfInt );
//				--maxStripes;
//			}
//			cmd.response.clear();
//			intAtoByteA( firstRecord, dataArray, 0 );
//			cmd.bArray = Arrays.copyOf(dataArray, dataArrayIdx);
//
//		}
//		else {
//			cmd.response.add("Valid Range: 1 to " + Integer.toString(maxReadStripes) );
//			cmd.response.add( ">");
//		}
//
//		cmd.action = 333;				// handled internally
	}


	@Override
	public boolean getStreamHeader( CmdStruct cmd ) {

		if( streamHeader != null && streamHeader.isValid ) {
			List<String> sList = streamHeader.getAsXML();
			for( String s : sList )
				cmd.response.add( s );
		}
		else
			cmd.response.add( "Header Not Available" );

		return true;
	}


	private List<String> decodeRecChannelsResponse(List<String> cmdResponse) {
		ArrayList<String> headerStrs = new ArrayList<String>();

		headerStrs.add("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>");
		headerStrs.add("<header>");
			headerStrs.add("<version>V3</version>");

			String mainPeriodStr;												// will always be number of us
			if( basePPM.isResamplingActive() )
				mainPeriodStr = Integer.toString(basePPM.requestedResampleuS) + "us";
			else
				mainPeriodStr = Integer.toString( basePPM.devicePerioduS ) + "us";

			headerStrs.add( "<devicePerioduS>" + Integer.toString( 131072 ) + "us" + "</devicePerioduS>" );

			headerStrs.add( "<mainPeriod>" + "131072us" + "</mainPeriod>" );
			headerStrs.add( "<legacyVersion>" + Integer.toString(99) + "</legacyVersion>" );
			headerStrs.add( "<legacyFormat>" + Integer.toString(99) + "</legacyFormat>" );
			headerStrs.add( "<legacyAverage>" + Integer.toString(32 * 1024) + "</legacyAverage>" );
			headerStrs.add( "<channels>" );

			// No inferred Status channel for mom basePPM.addChannelStrings( headerStrs, "Status", "status", "NA", 0, 0 );

				int dataPosition = 1;
				for( String str : cmdResponse ) {
					String[] parts = str.split(",");
					if( parts.length < 4 )
						continue;
					String groupName;
					if( parts[1].endsWith("V") )
						groupName = "voltage";
					else
						if( parts[1].endsWith("A") )
							groupName = "current";
						else
							groupName = parts[1];				// need to add power calc, available channels to be based on the mode settings at this time (if 12vV and 12vA and power calc then... Also need to modify Power units to uW or similar
					try {
						int bitWidth = Integer.parseInt( parts[3] );
						long maxValue = (long) Math.pow(2, bitWidth -1);
						basePPM.addChannelStrings( headerStrs, parts[0], groupName, parts[1], maxValue, dataPosition++ );
					}
					catch( Exception e ) {

					}
				}

			headerStrs.add( "</channels>" );

		headerStrs.add( "</header>" );

		return headerStrs;
	}


	/*
	 * old buffer does not track groups and will likely throw exceptions when groups appear out of ordered sequence
	 */
	private final boolean useOldBuffer = false;


	private PagedBuffer pagedBuffer = new PagedBuffer();
	private BufferPage binaryStreamDataBuffer = null;
	private BufferPage textStreamDataBuffer = null;


	private void cmdStreamText(CmdStruct cmd, int maxReadStripes) {		// from HD
		int maxStripes = -1;
//	        DebugUtil.debugMsgln("Buffer Start");

		// command begins with "stream text" as verified by caller
		//

		// for ppms this is read at stream start to ensure the settings are fully up to date...
		// But, in the case of reliable fixture connect/disconnect detection, use of the rec channels? command does make sense and will be reliable over the duration of a connect event
		//
		if( cmd.command.substring(12).trim().startsWith("header") ) {
			/*
			 * example response to command command rec channels?
			 * 	FAIL: No channel data available, check Fixture connection
			 * OR
				12V,V,0xF10
				12V,A,0xE19
				3.3V,V,0xF10
				3.3V,A,0xE19
				3.3V Aux,V,0xF10
				3.3V Aux,A,0xE19
			*/

			getStreamHeader( cmd );
			cmd.response.add( ">");
		}
		else {
			cmd.debugFilterLongReply = true;
			if( cmd.command.substring(12).trim().startsWith("all") )
				maxStripes = maxReadStripes;
			else {
				try {
					maxStripes = Integer.parseInt(cmd.command.substring(12).trim());
				}
				catch( NumberFormatException e ) {
					maxStripes = -1;
				}
				catch( StringIndexOutOfBoundsException e) {
					maxStripes = -1;
				}
			}

			/*
			 * get buffer page set limit and position as necessary and pass to some ToText method , eof and eod are still required
			 * Once a buffer has been claimed it will be out of the scope of binary transfer
			 */

			if( (maxStripes > 0) && (maxStripes <= maxReadStripes) ) {
				if( useOldBuffer ) {
					while( maxStripes > 0 ) {
						StreamRawData rawData = streamBuffer.getOldestStripe();			// streamBuffer.getSimulatedStripe( 500, 6 );
						if( rawData == null ) {
							cmd.response.add( "eof" );
							break;
						}

						cmd.response.add( rawData.asText() );
						--maxStripes;
					}
					cmd.response.add( ">");
				}
				else {
					cmd.stringBuilder = null;
					cmd.response.add( ">");														// dummy terminator
					final StringBuilder sb = new StringBuilder();

					while( maxStripes > 0 ) {
						// if( running ) we have data, else we have stopped.
						if( textStreamDataBuffer == null ) {
							textStreamDataBuffer = pagedBuffer.getFullBufferPage();

							if( textStreamDataBuffer != null ) {
								//final int version =
								textStreamDataBuffer.getData().getInt();				// skip header int
								textStreamDataBuffer.skipEntryCounts();					// skip to data
							}
						}

						if( textStreamDataBuffer == null ) {
							sb.append( "eof\r\n" );								// could add if not running and end of stream specifier
							break;
						}

						bufferStripeToStringBuilder( textStreamDataBuffer, sb );
						sb.append( "\r\n" );

						--maxStripes;
						++recordCount;

						if( !textStreamDataBuffer.getData().hasRemaining() )
							textStreamDataBuffer = null;
					}

					sb.append(">");
					cmd.stringBuilder = sb;
				}


	//		        DebugUtil.debugMsgln("Buffer Done");
			}
			else {
				cmd.response.add("Valid Range: 1 to " + Integer.toString(maxReadStripes) );
				cmd.response.add( ">");
			}
		}
		cmd.action = 333;				// handled internally
	}


	StringBuilder stripeStringBuilder = new StringBuilder();

	private void bufferStripeToStringBuilder( BufferPage textStreamDataBuffer, StringBuilder sb ) {
		final ByteBuffer data = textStreamDataBuffer.getData();
		sb.append( recordCount );

		final int groupId = data.getShort();
		sb.append( " " );
		sb.append( groupId );

		dataGeneratorList.get(groupId).toStringBuilder( data, sb );
	}


	private class FixtureChannel {
		private static final String DigitalUnitsSpecifierString = "(Digital)";

		public final static int channelSizeInBytes = 6;

		private final String name;
		private final String units;
		final String group;

		private final int position;				// position of this channel in the data block
		private final int groupId;
		private final int rawDataPrecision;
		private final byte dataWidth;					// bit width of data


		private final double dataPrecision;

		/**
		 *
		 * @param position
		 * @param data							// full data buffer
		 * @param channelOffset					// offset to start of this channel
		 * @param stringTableStartOffset		// offset to start of string table
		 */
		public FixtureChannel( int position, byte[] data, int channelOffset, int stringTableStartOffset ) {
			final ByteBuffer bb = ByteBuffer.wrap( data, channelOffset, channelSizeInBytes );
			bb.order(ByteOrder.LITTLE_ENDIAN);

			this.position = position;

			final short nameIdx = bb.getShort();
			final short unitsIdx = bb.getShort();

			dataWidth = bb.get();									// 8 bits of gpw = width. Little endian buffer swaps all bytes, so datawidth is here
			byte tempByte = bb.get();								// remaining 8 bits of gpw

			name = getStringAt( data, stringTableStartOffset + nameIdx );

			groupId = (tempByte >> 4) & 0x0f;

			if( unitsIdx < 0 || getStringAt( data, stringTableStartOffset + unitsIdx ).endsWith("D") ) {								// digital == 0xffff
				units = DigitalUnitsSpecifierString;								// unique string can be filtered when necessary
				group = "NA";

				rawDataPrecision = 0;												// maps to " ", ie not SI prefix
				dataPrecision = 1;
			}
			else {
				units = getStringAt( data, stringTableStartOffset + unitsIdx );
				if( units.endsWith("A") )							// frig units association across channels
					group = "current";
				else
					if( units.endsWith("V") )
						group = "voltage";
					else
//						if( units.endsWith("D") ) {
//							//units = "(Digital)";					// unique string can be filtered when necessary
//							group = "digital";
//						}
//						else
							group = "NA";

				tempByte = (byte)(tempByte  & 0x0f);

				if( (tempByte & 0x08) > 0 )							// deal with signed precision
					rawDataPrecision = (tempByte & 0x07) - 8;
				else
					rawDataPrecision = tempByte & 0x07;

				dataPrecision = Math.pow( 1000, rawDataPrecision );
			}
		}


		private String getStringAt( byte[] data, int startIdx  ) {
			final StringBuilder sb = new StringBuilder();

			byte cbyte = data[startIdx++];
			while( cbyte != 0 ) {
				sb.append( (char) cbyte );
				cbyte = data[startIdx++];
			}

			return sb.toString();
		}


		public List<String> getAsXML(int position) {
			List<String> retVal = new LinkedList<String>();

			retVal.add("<name>" + name + "</name>" );
			retVal.add("<group>" + group + "</group>" );

			if( units.equals("D"))
				retVal.add("<units>" + getSIPrefix() + DigitalUnitsSpecifierString + "</units>" );
			else
				retVal.add("<units>" + getSIPrefix() + units + "</units>" );

			retVal.add("<maxTValue>" + getMaxTValue() + "</maxTValue>" );

			retVal.add("<dataPosition>" + position + "</dataPosition>" );

			return retVal;
		}


		private String getMaxTValue() {
			return Long.toString( Math.round( Math.pow(2, dataWidth) ) -1 );
		}


		private final String[] unitModifiers = { "p", "n", "u", "m", "", "k", "M", "G" };

		private String getSIPrefix() {
			// lookup SI prefix
			int idx = rawDataPrecision + 4;
			if( idx < 0 || idx >= unitModifiers.length )
				return "?";

			return unitModifiers[idx];
		}


		protected int getGroupId() {
			return groupId;
		}
	}


	private class DataGenerator {

		public static final int byteSizeOfGroupId = 2;

		private final short groupId;

		private final int nChannels;

		private final List<ChannelData> channelDataList = new ArrayList<ChannelData>();

		private int rawDataBufferByteSize;


		public class ChannelData {
			private final byte[] bitOffsetMaskMap =  { 0, 1, 3, 7, 0x0f, 0x1f, 0x3f, 0x7f, (byte)0xff };

			static final int byteWidth = 8;
			private final int dataWidth;
			private final int bitOffset;

			private final int msbMask;
			private final int msbShift;			// msb right shift for dataWidth < 8
			private int lsbMask;
			private final int lsbShift;			// lsb right shift for ls data

			private final int maxUnsignedValue;
			private final int signBitMask;

			/**
			 *
			 * @param dataWidth			width of data field in bits
			 * @param bitOffset			1..8, bit number of the upper most binary digit in a byte
			 */
			public ChannelData( final int dWidth, final int bitOffset ) {
				dataWidth = dWidth;
				this.bitOffset = bitOffset;
				// from dataWidth and bitOffset we should be able to calc. mask and shift functions
				// shift is function of datawidth and bitoffset

				final int orphanbits =  Math.abs(dataWidth - bitOffset) % 8;					// bits remaining

				msbShift = Math.abs(dataWidth - bitOffset);
				msbMask = bitOffsetMaskMap[bitOffset];

				if( (byteWidth - orphanbits) >= 8 )
					lsbShift = 0;
				else
					lsbShift = byteWidth - orphanbits;

				maxUnsignedValue = (int)Math.pow(2, dataWidth);
				if( dataWidth == 1 )
					signBitMask = 0;
				else
					signBitMask = maxUnsignedValue >>> 1;										// logical (vs arithmetic signed) right shift, when anded with derived value 0 is unsigned value, not 0 is signed value

//				int x = (bitOffset + dataWidth) %  byteWidth;
			}


			protected int getData( byte[] buffer, int offset ) {							// could replace this with some funky lambdas
				long retVal = 0;															// may overflow int during manipulation
				int bitsConsumed = 0;

				// 1st (msb) byte
				retVal = buffer[offset++] & (msbMask & 0xff);								// & 0xff to avoid unwanted sign extend

				if( dataWidth == 1 ) {														// single bit
					retVal = retVal >> msbShift;											// correct alignment while destroying any RHS data
					retVal = retVal & bitOffsetMaskMap[dataWidth];
					return (int) retVal;
				}

				if( (dataWidth < byteWidth) && (byteWidth - dataWidth <= bitOffset) ) {		// dataWidth contained within size of a byte and bitOffset confirms all data bits are within this byte
					retVal = retVal >> msbShift;											// correct alignment while destroying any RHS data
					return (int) retVal;
				}

				// intermediate and final bytes
				bitsConsumed = bitOffset;
				while( dataWidth - bitsConsumed > 0) {
					retVal = retVal << 8;
					retVal = retVal | (buffer[offset++] & 0xff);
					bitsConsumed += byteWidth;
				}

				retVal = retVal >> lsbShift;

				if( (retVal & signBitMask) != 0 )
					retVal = retVal - maxUnsignedValue;

				return (int) retVal;
			}


			public int getBitsConsumed() {
				return dataWidth;
			}


			/**
			 * @return the minimum number of bytes required to hold dataWidth bits
			 */
			public int getBytesConsumed() {
				return (dataWidth / byteWidth) + 1;
			}


			int debugOffset = 0;				// debugging, device data values corrupted by this code!

			/*
			 * populate ByteBuffer with group data from buffer
			 */
			public void getData(byte[] buffer, int offset, ByteBuffer dataBB ) {

				final int dataValue = (int)( getData( buffer, offset ) *  (1 + (debugOffset / 1000.0)) );
				final int bytesConsumed = getBytesConsumed();

				if( bytesConsumed > 2 )
					dataBB.putInt( dataValue );
				else
					if( bytesConsumed > 1 )
						dataBB.putShort( (short) dataValue );
					else
						dataBB.put( (byte) (dataValue + debugOffset));

				if( ++debugOffset > 1000 )
					debugOffset = 0;
			}


			public void toStringBuffer( ByteBuffer dataBB, StringBuilder sb ) {
				final int bytesConsumed = getBytesConsumed();

				if( bytesConsumed > 2 )
					sb.append( dataBB.getInt() );
				else
					if( bytesConsumed > 1 )
						sb.append( dataBB.getShort() );
					else
						sb.append( dataBB.get() );
			}

		}


		public DataGenerator( short groupId, final List<FixtureChannel> channels ) {
			this.groupId = groupId;

			nChannels = channels.size();
			int bitOffset = 8;
			int totalBitSize = 0;


			for( FixtureChannel channel : channels ) {
				channelDataList.add( new ChannelData( channel.dataWidth, bitOffset ) );
				totalBitSize += channel.dataWidth;
				bitOffset = ChannelData.byteWidth - ((totalBitSize) % ChannelData.byteWidth);
			}

			if( totalBitSize % ChannelData.byteWidth > 0 )
				rawDataBufferByteSize = (totalBitSize / ChannelData.byteWidth) + 1;
			else
				rawDataBufferByteSize = (totalBitSize / ChannelData.byteWidth);

			// not here, groupId added to data size only relevant to a destination/transfer buffer rawDataBufferByteSize += byteSizeOfGroupId;
		}


		public void toStringBuilder(ByteBuffer data, StringBuilder sb) {
			for( ChannelData channelData : channelDataList ) {
				sb.append( " " );
				channelData.toStringBuffer( data, sb );
			}
		}


		/**
		 * size in bytes of raw data buffer required to produce one full data stripe
		 * @return
		 */
		public int getRawDataBufferByteSize() {
			return rawDataBufferByteSize;								// function of bits/width in list
		}


		/**
		 *
		 * @return number of data elements/channels
		 */
		public int getDataElementCount() {
			return nChannels;
		}


		/**
		 *
		 * @param blockId
		 * @param buffer			(buffer size - startOffset) must be >= getRawDataBufferByteSize()
		 * @param startOffset		startOffset allows protocol/preamble etc to be skipped
		 * @param data				array int[ getDataElementCount() ] to be populated
		 */
		public void getData( byte[] buffer, final int startOffset, int[] data ) {
			//int offset = startOffset;
			int bitsConsumed = 0;
			int i = 0;

			for( ChannelData channelData : channelDataList ) {
				data[i++] = channelData.getData( buffer, startOffset + (bitsConsumed / ChannelData.byteWidth) );
				bitsConsumed = bitsConsumed + channelData.getBitsConsumed();										// keep a running total allowing correct calculation of bytes consumed (bitsConsumed / ChannelData.byteWidth)
				//offset = offset + channelData.getBytesConsumed();
			}
		}


		/*
		 * populate ByteBuffer with group data:
		 * 		groupId	:	short
		 * 		group data
		 */
		public void getData( byte[] buffer, final int startOffset, BufferPage dataBuffer ) {
			final ByteBuffer dataBB = dataBuffer.getData();

			int bitsConsumed = 0;
			int i = 0;

			dataBB.putShort(groupId);

			for( ChannelData channelData : channelDataList ) {
				channelData.getData( buffer, startOffset + (bitsConsumed / ChannelData.byteWidth), dataBB );
				bitsConsumed = bitsConsumed + channelData.getBitsConsumed();										// keep a running total allowing correct calculation of bytes consumed (bitsConsumed / ChannelData.byteWidth)
			}
		}
	}

	private class StreamHeader {

		final static int fixedHeaderByteSize = 2 + 18;									// 2 byte stream header + internalStatus..stringTableSize

		final int version;
		final int average;

		boolean isValid = false;
		private final short internalStatus;
		private final short fixturePartNo;
		private final short fixtureVersionNo;
		private final short headerVersion;
		private final short sampleRateBase;
		private final short sampleRateExponent;
		private final short channelCount;
		private final short channelTableSizeBytes;
		private final short stringTableSize;

		private final long timeBaseNS;

		private final double calcedSampleFrequency;
		private final int devicePerioduS;

		private final List<List<FixtureChannel>> channelsListList = new ArrayList<List<FixtureChannel>>();

		private int bytesConsumed;


		/**
		 * decode data[] into header fields. Test for fixture missing should be done by caller, though consideration is required for a late disconnect event
		 * Also, what if header is > 512 bytes?
		 *
		 * @param data
		 */
		public StreamHeader( byte[] data ) {

				ByteBuffer bb = ByteBuffer.wrap( data );			 //new ByteBuffer();
				bb.order(ByteOrder.LITTLE_ENDIAN);

				version = bb.get();
				average = bb.get();

				internalStatus = bb.getShort();
				fixturePartNo = bb.getShort();
				fixtureVersionNo = bb.getShort();
				headerVersion = bb.getShort();
				sampleRateBase = bb.getShort();
				sampleRateExponent = bb.getShort();
				channelCount = bb.getShort();
				channelTableSizeBytes = bb.getShort();
				stringTableSize = bb.getShort();

				timeBaseNS =  (long) ((long)sampleRateBase * Math.pow(10, sampleRateExponent));

				calcedSampleFrequency = 1.0/(double)timeBaseNS;

				devicePerioduS = (int)(Math.pow(2, average) * (calcedSampleFrequency * 1000000));

				//basePPM.devicePerioduS = (int)Math.pow(2, streamHeader.average) * 4;

				// register the channels and split by groupId
				for( int i = 0; i < channelCount; ++i )
					addToChannelList( new FixtureChannel( i, data, fixedHeaderByteSize + (i * FixtureChannel.channelSizeInBytes), fixedHeaderByteSize + channelTableSizeBytes ) );

				bytesConsumed = fixedHeaderByteSize + channelTableSizeBytes + stringTableSize;
				isValid = true;
		}


		public List<List<FixtureChannel>> getChannelsListList() {
			return channelsListList;
		}


		public int getGroupIdCount() {
			return channelsListList.size();
		}


		private void addChannelsList() {
			channelsListList.add( new ArrayList<FixtureChannel>());
		}


		private void addToChannelList( FixtureChannel fc ) {
			final int groupId = fc.groupId;

			while( groupId >= getGroupIdCount() )
				addChannelsList();

			getChannelsList( groupId ).add( fc );
		}


		public List<FixtureChannel> getChannelsList( int groupId ) {
			return channelsListList.get(groupId);
		}


		public List<FixtureChannel> getChannelsList() {
			return channelsListList.get(0);
		}


		/**
		 * returns the number of bytes consumed when creating the header
		 * @return
		 */
		public int getBytesConsumed() {
			return bytesConsumed;
		}


		public List<String> getAsXML() {
			List<String> retVal = new LinkedList<String>();
			retVal.add("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>");
			retVal.add("<header>");

				retVal.add("<version>V3</version>");
				retVal.add("<devicePeriod>" + devicePerioduS + "us</devicePeriod>");			// qps expects us in V3 header (? check if us/ns suffix works) so divide by 1000
				retVal.add("<mainPeriod>" + devicePerioduS + "us</mainPeriod>");				// qps expects us in V3 header so divide by 1000

				retVal.add("<legacyVersion>" + version + "</legacyVersion>");
				retVal.add("<legacyFormat>0</legacyFormat>");
				retVal.add("<legacyAverage>" + average + "</legacyAverage>");

				for( List<FixtureChannel> chanList : getChannelsListList() ) {
					retVal.add("<channels>");
					retVal.add("<groupId>" + chanList.get(0).groupId + "</groupId>" );

					/* Mom, no status
					retVal.add("<channel>");
						retVal.add("<name>Status</name>");					// do we need this for fixture?
						retVal.add("<group>status</group>");
						retVal.add("<units>NA</units>");
						retVal.add("<maxTValue>0</maxTValue>");
						retVal.add("<dataPosition>0</dataPosition>");
					retVal.add("</channel>");
					*/

					int position = 1;

					for( FixtureChannel channel : chanList ) {
						retVal.add("<channel>");
						retVal.addAll( channel.getAsXML( position++ ) );
						retVal.add("</channel>");
					}

				retVal.add("</channels>");

				}

			retVal.add("</header>");

			return retVal;
		}

	}




	private int streamDataIdx;
	private byte[] streamData = null;

	private StreamHeader streamHeader;

	private boolean processHeader() {
		streamHeader = new StreamHeader( streamData );

		if( !streamHeader.isValid )
			streamHeader = null;

		return streamHeader != null;
	}




// local class needed	StreamHeader streamHeader;

	private boolean isHeaderValid() {
		boolean retVal = false;

		if( streamData.length > 1 )
			//if( streamData[0] == 5 )				// what to check here
				retVal = true;

		return retVal;
	}


	private List<DataGenerator> dataGeneratorList = new ArrayList<DataGenerator>();
	private List<HoldingBuffer> holdingBufferList = new ArrayList<HoldingBuffer>();		// not used ????
	private List<int[]> dataStripeList = new ArrayList<int[]>();

	private int largestGroupOutSizeBytes = -1;

//	private DataGenerator dataGenerator = null;
//	private HoldingBuffer holdingBuffer = null;
//	private int [] dataStripe = null;											// need arraylist indexed by group, size of arrays will match fixture header definition

	private void setDecodeParameters() {

		dataGeneratorList.clear();
		holdingBufferList.clear();
		dataStripeList.clear();

		for( int i = 0; i < streamHeader.getGroupIdCount(); ++i ) {
			final DataGenerator dataGenerator = new DataGenerator( (short) i, streamHeader.getChannelsList(i) );				// will need one per group, selection of which to be driven by processData code
			final HoldingBuffer holdingBuffer = new HoldingBuffer( dataGenerator.getRawDataBufferByteSize() );		// will need more advanced holding iff multiple data groups are to be supported

			final int[] dataStripe = new int[ dataGenerator.getDataElementCount() ];			// could optimise as using 32 bits to represent a single bit!

			dataGeneratorList.add(dataGenerator);
			holdingBufferList.add(holdingBuffer);
			dataStripeList.add(dataStripe);

			final int groupOutSizeBytes = (dataGenerator.getDataElementCount() * 4)  + DataGenerator.byteSizeOfGroupId;	// correct this to handle non-32bit int fields

			largestGroupOutSizeBytes = Math.max(groupOutSizeBytes, largestGroupOutSizeBytes);
		}
	}


	protected StreamState streamState;

	private boolean running = false;		// from HD
	private long recordCount = 0;

	private boolean syncPacketActive = false;
	private boolean waitPacketAfterSync = false;

	private final int syncRetryCountReset = 10;

	private int syncRetryCount = syncRetryCountReset;

	private void handleOneByteStatus( byte b ) {
//System.out.println( "one byte " + b );
		switch( b ) {

			case 0 :
				//user exit
				stopStreamByStatusCode( StreamStatusCodes.StoppedUser );
//				statusCode = 0;
				DebugUtil.debugMsgln("User Exit");
				break;

			case 1 :

				//Overrrun
				stopStreamByStatusCode( StreamStatusCodes.StoppedOverrun );
//				statusCode = -1;
				DebugUtil.debugMsgln("Overrrun");
				break;

			case 2 :

				//Over Temperature
				stopStreamByStatusCode( StreamStatusCodes.StoppedOverTemp );
//				statusCode = -2;
				DebugUtil.debugMsgln("Over Temperature");
				break;

			case 3 :

				//No valid data
//				statusCode = -3;
				DebugUtil.debugMsgln("No Valid Data");
				break;

			case 7 :
				syncRetryCount = syncRetryCountReset;
				myDle.deviceInfo.cmdLock.release();					// permit commands during sync period
				DebugUtil.debugMsg("7");
//				try {
//					Thread.sleep(1);
//				} catch (InterruptedException e) {
//					// TODO Auto-generated catch block
//					e.printStackTrace();
//				}
				syncPacketActive = true;
				break;

			case 8 : 
				// fixture disconnected
				// do something
				stopStreamByStatusCode( StreamStatusCodes.StopppedFixtureDisconnect );
				DebugUtil.debugMsgln("Fixture Disconnect");
				break;
				
			case 99 :
				// waitPacketAfterSync was set when the sync packet was sent. If it is still true when this timeout occurs we retry the sync send
				//if( waitPacketAfterSync && ( --syncRetryCount > 0  ) ) {
				if( --syncRetryCount > 0  ) {
					if( waitPacketAfterSync ) {
						// this was an attempt to flush the device command queue to avoid a fail on the next command...
						// if we arrive here by means of a missed sync myDle.deviceInfo.cmdLock remains in acquire state,
						myDle.deviceInfo.cmdLock.release();
						CmdStruct frigCmd = new CmdStruct();
						frigCmd.command = "hello";
						myDle.deviceInfo.executeCommand(frigCmd);
						System.out.println( "sync unblock " + (frigCmd.response.size() > 0 ? frigCmd.response.get(0) : "no response") );
						myDle.deviceInfo.cmdLock.acquireUninterruptibly();
						System.out.println( "lock acquired");
					}
					sendSynk();											// cmd path will be closed if this is accepted by the device
		/////////			System.out.println( "sendSync" );
					try {
						Thread.sleep(10);
					} catch (InterruptedException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}

				}
				else {
					syncRetryCount = syncRetryCountReset;
					//if( waitPacketAfterSync )													// timeout, attempt a sync ack to unblock stream
					//	myDle.deviceInfo.cmdLock.release();					// (permit commands during sync period)
					DebugUtil.debugMsg("99");								// under normal circumstances this should never happen over USB,
					//syncPacketActive = true;							// but in testing it does on occasion, (related to wireshark or other system activity?)
			        // timeout, abort stream
			        streamStatusCode = StreamStatusCodes.StoppedCommsFailure;
			        disableStreamReceive();
			        myDle.deviceInfo.cmdLock.release();
				}
				break;												// maybe the addition ack will unblock the pipe?

		}
	}


	public void stopStreamByStatusCode( StreamStatusCodes sCode ) {
		streamStatusCode = sCode;
		disableStreamReceive();
		myDle.deviceInfo.cmdLock.release();					// ending, re-enable commands
		running = false;
	}


	class HoldingBuffer {
		private final byte[] buffer;
		private boolean inUse = false;
		private int holdingLength;

		public HoldingBuffer( int sizeBytes ) {
			buffer = new byte[sizeBytes];
		}


		/**
		 * copies len bytes from data[startIdx] into holding buffer
		 * @param data
		 * @param startIdx
		 */
		public void setBuffer( byte[] data, int startIdx, int len ) {
			holdingLength = len;
			inUse = true;
			System.arraycopy(data, startIdx, buffer, holdingLength, holdingLength);
		}

		public byte[] getBuffer() {
			inUse = false;
			return buffer;
		}

		public boolean isInUse() {
			return inUse;
		}


		public int completeBuffer(byte[] streamData) {
			final int retVal = buffer.length - holdingLength;
			System.arraycopy(streamData, 0, buffer, holdingLength, retVal);
			return retVal;
		}
	}



	private void processData() {
		int dataLen = streamData.length;
		streamDataIdx = 0;
		boolean dataReady = false;

		if(waitPacketAfterSync && dataLen > 0) {
			//syncPacketActive = false;
			waitPacketAfterSync = false;
		}

		if( (streamData.length & 1) != 0 ) {
			// decode status byte(s)
			if( DebugUtil.isEnableDebug() ) {
				String s = "";
				for( byte b : streamData )
					s = s + Byte.toString(b) + " ";
				DebugUtil.debugMsgln( s );
			}
			handleOneByteStatus( streamData[streamData.length - 1] );				// status byte is the last byte in the buffer
			--dataLen;																// effectively remove the status byte
			dataReady = true;
		}

		if( dataLen > 0 ) {
			int i = 1;
			while( i < dataLen ) {						// sad that we need to do this but swap those bytes. This is a lot of data processing which could be done in the FPGA<>fixture chain :(
				final byte b = streamData[i];
				streamData[i] = streamData[i-1];
				streamData[i-1] = b;
				i = i + 2;
			}

			while( streamDataIdx < dataLen ) {
				//
				// agreed with mike:
				//		Data will NOT span 512 byte buffers.					// possibly inefficient
				//		Each data packet will start on a byte boundary
				//
				final byte frameId = streamData[ streamDataIdx ];

				// future: frameId 0x0A is a trigger packet

				if( frameId != 4 ) {		// frameId == 8 || frameId == -1 ) {	// 8 is end of packet data, -1 is the packet filler that may be seen in case of combined packets
					++streamDataIdx;
					continue;				// for padding frame, we can assume this is end of data for this buffer.
											// Currently we need to process all remaining bytes, as the fpga always gives us 512 byte buffers(!), in case TCP has combined  two packets due to
											// a retransmition (seen in over night runs)
				}

				final int groupId = streamData[ streamDataIdx +1 ];

				if( streamDataIdx + dataGeneratorList.get(groupId).getRawDataBufferByteSize() < dataLen ) {
					// need to consider frame id. 4 -> data packet
					if( frameId == 4 ) {
						if( useOldBuffer ) {
							//streamDataIdx = streamDataIdx + 3;												// skip over data header [frame id][block id] + SeqNo. frame id: not doing anything with it for now,  block id: for now is always 0
							streamDataIdx = streamDataIdx + 2;													// skip over data header [frame id][group id]. frame id: data or no more data group id: matches header definition
							dataGeneratorList.get(groupId).getData( streamData, streamDataIdx, dataStripeList.get(groupId) );
							//streamDataIdx = streamDataIdx + dataGenerator.getRawDataBufferByteSize() + 1;		// +1 for stupid crc8
							streamDataIdx = streamDataIdx + dataGeneratorList.get(groupId).getRawDataBufferByteSize();			// crc8 has been removed
							streamBuffer.add( false, dataStripeList.get(groupId) );
						}
						else {
							final int spaceRequired = largestGroupOutSizeBytes;		// dataGeneratorList.get(groupId).getRawDataBufferByteSize();

							if( binaryStreamDataBuffer == null ) {
								binaryStreamDataBuffer = pagedBuffer.getFreeBufferPage();
								//
								// make space for header
								//
								if( binaryStreamDataBuffer == null ) {				// out of space
System.out.println(  System.currentTimeMillis() + " Out Of Space" );
pagedBuffer.printDebugInfo();

									break;											// ignore data and exit loop
								}

								binaryStreamDataBuffer.getData().putInt( 5 );		// header version 5
								binaryStreamDataBuffer.getData().putInt( -1 );		// # bytes in this buffer when passed down the line

								// allocate space for group entry counts, int32 one per group. Increment the corresponding value each time group data is added
								binaryStreamDataBuffer.createEntryCounts( dataGeneratorList.size() );
System.out.print( "n" );	// "New Buffer" );

							}

							// add data.
							streamDataIdx = streamDataIdx + 2;																	// skip over data header [frame id][group id]. frame id: data or no more data group id: matches header definition

							dataGeneratorList.get(groupId).getData( streamData, streamDataIdx, binaryStreamDataBuffer );
							binaryStreamDataBuffer.incEntryCounter( groupId );													// track number of entries per groupId

							streamDataIdx = streamDataIdx + dataGeneratorList.get(groupId).getRawDataBufferByteSize();			// crc8 has been removed


							// do we have space for the worst case next packet?
							// assuming the buffer size is >> stripe size this should not be too inefficient NB as of now largestGroupOutSizeBytes requires correction
							if( !binaryStreamDataBuffer.isSpaceFor(spaceRequired) ) {
								pagedBuffer.qFullBufferPage( binaryStreamDataBuffer );

binaryStreamDataBuffer.getData().rewind();
System.out.println( System.currentTimeMillis() + " Buffer to FullPageQ: " + binaryStreamDataBuffer.getData().getShort() + " " + binaryStreamDataBuffer.getData().getShort() + " I" + binaryStreamDataBuffer.getData().getInt() + " " + binaryStreamDataBuffer.getData().getShort());

								binaryStreamDataBuffer = null;

							}

						}
					}
				}
//				}
//				else {
//					// hold data over to next packet
//					streamDataIdx = streamDataIdx + 2;
//					holdingBuffer.setBuffer( streamData, streamDataIdx, dataLen - streamDataIdx );
//					streamDataIdx = dataLen;
//				}
			}
			dataReady = true;
		}

		if( pagedBuffer.isFullPageRequested() ) {											// only do this after streamData consumed
System.out.println(  System.currentTimeMillis() + " Page Requested" );
			if( binaryStreamDataBuffer != null && !binaryStreamDataBuffer.isEmpty() ) {
				pagedBuffer.qFullBufferPage( binaryStreamDataBuffer );						// buffer may not be full but send it anyway else 32k averaging will have nothing to return for a long time

binaryStreamDataBuffer.getData().rewind();
System.out.println(  System.currentTimeMillis() + " Page Requested : Queued: " + binaryStreamDataBuffer.getData().getShort() + " " + binaryStreamDataBuffer.getData().getShort() + " I" +
												binaryStreamDataBuffer.getData().getInt() + " " + binaryStreamDataBuffer.getData().getShort() + " " +
												binaryStreamDataBuffer.getData().getShort() + " " + binaryStreamDataBuffer.getData().getShort() + " " +
												binaryStreamDataBuffer.getData().getShort() + " " + binaryStreamDataBuffer.getData().getShort() );
				binaryStreamDataBuffer = null;
			}
System.out.println(  System.currentTimeMillis() + " No Page Ready" );
		}


		if( syncPacketActive ) {
			//if( !waitPacketAfterSync )
				myDle.deviceInfo.cmdLock.acquireUninterruptibly();				// wait for any current cmd and block future cmds during hd stream
			sendSynk();
			waitPacketAfterSync = true;											// will be cleared on next data packet
		}
		else
			if( dataReady && running )
				sendStreamContinue();

		if( streamBase != null )
			streamBase.fireDataReadyEvent( streamStatusCode, streamBuffer.size() );

		return;
	}



	@Override
	public void streamEvent(DeviceDataBuffer ddBuff) {
		streamData = ddBuff.buffer;							// code from HD
		switch( streamState ) {
			case waitHeader:
				if( isHeaderValid() ) {
					if( processHeader() ) {
						setDecodeParameters();

// TODO need more complex implementation for multiple groups!						basePPM.sanityCheckResample();
// TODO need more complex implementation for multiple groups!						streamBuffer.configureResampling( basePPM.isResamplingActive(), basePPM.devicePerioduS, basePPM.requestedResampleuS, dataGenerator.getDataElementCount() );

						streamState = StreamState.waitData;
						streamHeader.isValid = true;
					}
				}
				if( (streamState == StreamState.waitData) &&  streamHeader.isValid )
					streamData = Arrays.copyOfRange(ddBuff.buffer, streamHeader.getBytesConsumed(), ddBuff.buffer.length);
//					// and fall through
				else
					break;

			case waitData:
				processData();
				break;
//
			default:
				break;
		}
		return;
	}


	@Override
    public void addDataReadyListener(StreamDataReadyListener toAdd) {
        streamBase.addDataReadyListener(toAdd);
    }


	private void sendStreamContinue() {
		myDle.deviceInfo.sendStreamContinue();
	}


	private void sendSynk() {
		{
			// send syncReset
			DebugUtil.debugMsg("Send SyncReset ");

	        myDle.deviceInfo.sendSyncAck();									// current cmd complete, syncAck the hd ppm to continue the stream
	        DebugUtil.debugMsgln("Ack");
	        //waitPacketAfterSync = true;
	        syncPacketActive = false;
		}
	}

}
