Skip to content

TCAPData

Class for IQ Data TCAP format

xaratustrah@github Aug-2015

TCAPData

Bases: IQBase

Source code in iqtools/tcapdata.py 
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
class TCAPData(IQBase):
    def __init__(self, filename, header_filename):
        super().__init__(filename)

        if not header_filename:
            log.info('No TCAP header filename provided.')

        self.header_filename = header_filename

        # Additional fields in this subclass
        self.tcap_scalers = None
        self.tcap_pio = None

        self.version = ''
        self.adc_range = 0
        self.block_count = 0
        self.block_size = 0
        self.frame_size = 0
        self.decimation = 0
        self.trigger_time = 0
        self.segment_blocks = 0

        self.fs = 10e6 / (2 ** self.decimation)  # usually fixed to 312500
        # center is usually fixed to 1.6e5

        self.read_header()

    def read(self, nframes=10, lframes=1024, sframes=0):
        self.read_samples(nframes * lframes, offset=sframes * lframes)

    def read_samples(self, nsamples, offset=0):
        """Reads TCAP data.

        TCAP format information:

            - Each file contains 15625 blocks equivalent of 27min 18 seconds of recording
            - Each block is 2^17=131072 BYTES of data + 88 bytes of header
            - Each sample is 4 bytes = 32bits (2 I + 2 Q bytes), hence each block contains
            32768 complex valued samples
            - Sampling frequency is 312500 sps, thus the data of a block are each 0.105s
            worth of data and a resolution frequency of 312500 / 32768 = 9.5 Hz per block
            - To double the frequency resolution one can take two consecutive blocks which
            mean 4.77 Hz for two consecutive blocks which is 0.210 s of time.
            - Either from one block or two blocks a frame can be created.
            - An FFT is done on each frame. 10 such FFTs can be averaged to reduce noise.
            e.g. a 2-block frame would correspond to 2.12s data.
            - 400 such averaged groups of 10 frames can be plotted on a single spectrogram
            - each picture of a single spectrogram would then correspond to 14m and 8 sec.

        Args:
            nsamples (int): Number of samples to read
            offset (int, optional): Starting sample. Defaults to 0.
        """        

        BLOCK_HEADER_SIZE = 88
        BLOCK_DATA_SIZE = 2 ** 17
        BLOCK_SIZE = BLOCK_HEADER_SIZE + BLOCK_DATA_SIZE

        filesize = os.path.getsize(self.filename)
        # each file contains 15625 blocks
        if not filesize == 15625 * BLOCK_SIZE:
            log.info(
                "File size does not match block sizes times total number of blocks. Aborting...")
            return

        # read header section
        with open(self.filename, 'rb') as f:
            tfp = f.read(12)
            pio = f.read(12)
            scalers = f.read(64)

        self.date_time = self.parse_tcap_tfp(tfp)
        self.tcap_pio = pio
        self.tcap_scalers = scalers

        data_section_size = self.frame_size - BLOCK_HEADER_SIZE
        n_iq_samples = data_section_size / 2 / 2  # two bytes for I and two bytes for Q
        self.nsamples_total = self.segment_blocks * n_iq_samples

        # 4 comes from 2 times 2 byte integer for I and Q
        total_n_bytes = 4 * nsamples
        start_n_bytes = 4 * offset

        ba = bytearray()
        try:
            with open(self.filename, 'rb') as f:
                f.seek(BLOCK_HEADER_SIZE + start_n_bytes)
                for i in range(total_n_bytes):
                    if not f.tell() % BLOCK_SIZE:
                        log.info(
                            'File pointer before jump: {}'.format(f.tell()))
                        log.info(
                            "Reached end of block {}. Now skipoing header of block {}!".format(
                                int(f.tell() / BLOCK_SIZE),
                                int(
                                    f.tell() / BLOCK_SIZE) + 1))
                        f.seek(88, 1)
                        log.info('File pointer after jump: {}'.format(f.tell()))
                    # using bytearray.extend is much faster than using +=
                    ba.extend(f.read(1))
        except:
            log.error('File seems to end here!')
            return

        log.info('Total bytes read: {}'.format(len(ba)))

        # big endian 16 bit for I and 16 bit for Q
        self.data_array = np.frombuffer(ba, '>i2')
        self.data_array = self.data_array.astype(np.float32)
        self.data_array = self.data_array * self.scale
        self.data_array = self.data_array.view(np.complex64)

    def read_block(self, block_no):
        """
        Read the specified block between 1 and 15625.
        """
        BLOCK_HEADER_SIZE = 88
        BLOCK_DATA_SIZE = 2 ** 17
        BLOCK_SIZE = BLOCK_HEADER_SIZE + BLOCK_DATA_SIZE

        try:
            with open(self.filename, 'rb') as f:
                f.seek((block_no - 1) * BLOCK_SIZE)
                tfp = f.read(12)
                pio = f.read(12)
                scalers = f.read(64)
                ba = f.read(131072)
        except:
            log.error('File seems to end here!')
            return

        self.date_time = self.parse_tcap_tfp(tfp)
        self.tcap_pio = pio
        self.tcap_scalers = scalers

        log.info('Total bytes read: {}'.format(len(ba)))

        # big endian 16 bit for I and 16 bit for Q
        self.data_array = np.frombuffer(ba, '>i2')
        self.data_array = self.data_array.astype(np.float32)
        self.data_array = self.data_array * self.scale
        self.data_array = self.data_array.view(np.complex64)
        return self.data_array

    def get_frame(self, first, second):
        """Make a frame by connecting two blocks

        Args:
            first: first frame
            second: second frame

        Returns:
            array: Connected frames
        """        
        array = np.zeros(2 * 32768, dtype=np.complex64)
        array[0:32768] = self.read_block(first)
        array[32768:] = self.read_block(second)
        return array

    def parse_binary_tcap_header(self, ba):
        """Parses binary header of TCAP

        Args:
            ba (bytearray): Binary header
        """        
        version = ba[0:8]
        center_freq_np = np.fromstring(ba[8:16], dtype='>f8')[0]
        center_freq = struct.unpack('>d', ba[8:16])[0]
        adc_range = struct.unpack('>d', ba[16:24])[0]
        data_scale = struct.unpack('>d', ba[24:32])[0]
        block_count = struct.unpack('>Q', ba[32:40])[0]
        block_size = struct.unpack('>I', ba[40:44])[0]
        frame_size = struct.unpack('>I', ba[44:48])[0]
        decimation = struct.unpack('>H', ba[48:50])[0]
        config_flags = struct.unpack('>H', ba[50:52])[0]
        trigger_time = ba[500:512]
        # self.fs = 10**7 / 2 ** decimation

    def parse_tcap_tfp(self, ba):
        """Parses the TFP Header of TCAP DAT Files. This information is coded in BCD. The
        following table was taken form the original TCAP processing files in C.

        | ** bit #      ** | ** 15 - 12       ** | ** 11 - 8        ** | ** 7 - 4         ** | ** 3 - 0         ** |
        |------------------|---------------------|---------------------|---------------------|---------------------|
        |  timereg[0]      |  not defined        |  not defined        |  status             |  days hundreds      |
        |  timereg[1]      |  days tens          |  days units         |  hours tens         |  hours units        |
        |  timereg[2]      |  minutes tens       |  minutes units      |  seconds tens       |  seconds units      |
        |  timereg[3]      |  1E-1 seconds       |  1E-2 seconds       |  1E-3 seconds       |  1E-4 seconds       |
        |  timereg[4]      |  1E-5 seconds       |  1E-6 seconds       |  1E-7 seconds       |  not defined        |

         here we read the first 12 bytes ( 24 nibbles ) in the tfp byte array list. First 2 bytes
         should be ignored.

        Args:
            ba (bytearray): Binary header

        Returns:
            datetime: Time stamp
        """        
        tfp = list(ba)

        dh = (tfp[3] >> 0) & 0x0f

        dt = (tfp[4] >> 4) & 0x0f
        du = (tfp[4] >> 0) & 0x0f

        ht = (tfp[5] >> 4) & 0x0f
        hu = (tfp[5] >> 0) & 0x0f

        mt = (tfp[6] >> 4) & 0x0f
        mu = (tfp[6] >> 0) & 0x0f

        st = (tfp[7] >> 4) & 0x0f
        su = (tfp[7] >> 0) & 0x0f

        sem1 = (tfp[8] >> 4) & 0x0f
        sem2 = (tfp[8] >> 0) & 0x0f
        sem3 = (tfp[9] >> 4) & 0x0f
        sem4 = (tfp[9] >> 0) & 0x0f

        sem5 = (tfp[10] >> 4) & 0x0f
        sem6 = (tfp[10] >> 0) & 0x0f
        #sem7 = (tfp[11] >> 4) & 0x0f

        year = int(self.file_basename[0:4])
        days = int(dh * 100 + dt * 10 + du)
        hours = int(ht * 10 + hu)
        minutes = int(mt * 10 + mu)
        seconds = int(st * 10 + su)
        microseconds = int(1000 * (sem1 * 1e-1 + sem2 * 1e-2 + sem3 *
                                   1e-3 + sem4 * 1e-4 + sem5 * 1e-5 + sem6 * 1e-6))
        ts = datetime.datetime(year, 1, 1, hours, minutes, seconds,
                               microseconds) + datetime.timedelta(days - 1)
        return ts.strftime('%Y-%m-%d %H:%M:%S')

    def read_header(self):
        """Parses text header part.

        Returns:
            dictionary: Dictionary of values
        """        
        dic = {}
        with open(self.header_filename) as f:
            for line in f:
                name, var = line.split()
                dic[name.strip()] = var
        self.version = dic['version']
        self.center = float(dic['center_freq'])
        self.adc_range = float(dic['adc_range'])
        self.scale = float(dic['data_scale'])
        self.block_count = int(dic['block_count'])
        self.block_size = int(dic['block_size'])
        self.frame_size = int(dic['frame_size'])
        self.decimation = int(dic['decimation'])
        self.trigger_time = float(dic['trigger_time'])
        self.segment_blocks = int(dic['segment_blocks'])
        return dic

get_frame(first, second)

Make a frame by connecting two blocks

Parameters:

Name Type Description Default
first

first frame

 required
second

second frame

 required

Returns:

Name Type Description
array

Connected frames
Source code in iqtools/tcapdata.py 
159
160
161
162
163
164
165
166
167
168
169
170
171
172
def get_frame(self, first, second):
    """Make a frame by connecting two blocks

    Args:
        first: first frame
        second: second frame

    Returns:
        array: Connected frames
    """        
    array = np.zeros(2 * 32768, dtype=np.complex64)
    array[0:32768] = self.read_block(first)
    array[32768:] = self.read_block(second)
    return array

parse_binary_tcap_header(ba)

Parses binary header of TCAP

Parameters:

Name Type Description Default
ba bytearray

Binary header

 required
Source code in iqtools/tcapdata.py 
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
def parse_binary_tcap_header(self, ba):
    """Parses binary header of TCAP

    Args:
        ba (bytearray): Binary header
    """        
    version = ba[0:8]
    center_freq_np = np.fromstring(ba[8:16], dtype='>f8')[0]
    center_freq = struct.unpack('>d', ba[8:16])[0]
    adc_range = struct.unpack('>d', ba[16:24])[0]
    data_scale = struct.unpack('>d', ba[24:32])[0]
    block_count = struct.unpack('>Q', ba[32:40])[0]
    block_size = struct.unpack('>I', ba[40:44])[0]
    frame_size = struct.unpack('>I', ba[44:48])[0]
    decimation = struct.unpack('>H', ba[48:50])[0]
    config_flags = struct.unpack('>H', ba[50:52])[0]
    trigger_time = ba[500:512]

parse_tcap_tfp(ba)

Parses the TFP Header of TCAP DAT Files. This information is coded in BCD. The following table was taken form the original TCAP processing files in C.

** bit # ** ** 15 - 12 ** ** 11 - 8 ** ** 7 - 4 ** ** 3 - 0 **
timereg[0] not defined not defined status days hundreds
timereg[1] days tens days units hours tens hours units
timereg[2] minutes tens minutes units seconds tens seconds units
timereg[3] 1E-1 seconds 1E-2 seconds 1E-3 seconds 1E-4 seconds
timereg[4] 1E-5 seconds 1E-6 seconds 1E-7 seconds not defined

here we read the first 12 bytes ( 24 nibbles ) in the tfp byte array list. First 2 bytes should be ignored.

Parameters:

Name Type Description Default
ba bytearray

Binary header

 required

Returns:

Name Type Description
datetime

Time stamp
Source code in iqtools/tcapdata.py 
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
def parse_tcap_tfp(self, ba):
    """Parses the TFP Header of TCAP DAT Files. This information is coded in BCD. The
    following table was taken form the original TCAP processing files in C.

    | ** bit #      ** | ** 15 - 12       ** | ** 11 - 8        ** | ** 7 - 4         ** | ** 3 - 0         ** |
    |------------------|---------------------|---------------------|---------------------|---------------------|
    |  timereg[0]      |  not defined        |  not defined        |  status             |  days hundreds      |
    |  timereg[1]      |  days tens          |  days units         |  hours tens         |  hours units        |
    |  timereg[2]      |  minutes tens       |  minutes units      |  seconds tens       |  seconds units      |
    |  timereg[3]      |  1E-1 seconds       |  1E-2 seconds       |  1E-3 seconds       |  1E-4 seconds       |
    |  timereg[4]      |  1E-5 seconds       |  1E-6 seconds       |  1E-7 seconds       |  not defined        |

     here we read the first 12 bytes ( 24 nibbles ) in the tfp byte array list. First 2 bytes
     should be ignored.

    Args:
        ba (bytearray): Binary header

    Returns:
        datetime: Time stamp
    """        
    tfp = list(ba)

    dh = (tfp[3] >> 0) & 0x0f

    dt = (tfp[4] >> 4) & 0x0f
    du = (tfp[4] >> 0) & 0x0f

    ht = (tfp[5] >> 4) & 0x0f
    hu = (tfp[5] >> 0) & 0x0f

    mt = (tfp[6] >> 4) & 0x0f
    mu = (tfp[6] >> 0) & 0x0f

    st = (tfp[7] >> 4) & 0x0f
    su = (tfp[7] >> 0) & 0x0f

    sem1 = (tfp[8] >> 4) & 0x0f
    sem2 = (tfp[8] >> 0) & 0x0f
    sem3 = (tfp[9] >> 4) & 0x0f
    sem4 = (tfp[9] >> 0) & 0x0f

    sem5 = (tfp[10] >> 4) & 0x0f
    sem6 = (tfp[10] >> 0) & 0x0f
    #sem7 = (tfp[11] >> 4) & 0x0f

    year = int(self.file_basename[0:4])
    days = int(dh * 100 + dt * 10 + du)
    hours = int(ht * 10 + hu)
    minutes = int(mt * 10 + mu)
    seconds = int(st * 10 + su)
    microseconds = int(1000 * (sem1 * 1e-1 + sem2 * 1e-2 + sem3 *
                               1e-3 + sem4 * 1e-4 + sem5 * 1e-5 + sem6 * 1e-6))
    ts = datetime.datetime(year, 1, 1, hours, minutes, seconds,
                           microseconds) + datetime.timedelta(days - 1)
    return ts.strftime('%Y-%m-%d %H:%M:%S')

read_block(block_no)

Read the specified block between 1 and 15625.

Source code in iqtools/tcapdata.py 
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
def read_block(self, block_no):
    """
    Read the specified block between 1 and 15625.
    """
    BLOCK_HEADER_SIZE = 88
    BLOCK_DATA_SIZE = 2 ** 17
    BLOCK_SIZE = BLOCK_HEADER_SIZE + BLOCK_DATA_SIZE

    try:
        with open(self.filename, 'rb') as f:
            f.seek((block_no - 1) * BLOCK_SIZE)
            tfp = f.read(12)
            pio = f.read(12)
            scalers = f.read(64)
            ba = f.read(131072)
    except:
        log.error('File seems to end here!')
        return

    self.date_time = self.parse_tcap_tfp(tfp)
    self.tcap_pio = pio
    self.tcap_scalers = scalers

    log.info('Total bytes read: {}'.format(len(ba)))

    # big endian 16 bit for I and 16 bit for Q
    self.data_array = np.frombuffer(ba, '>i2')
    self.data_array = self.data_array.astype(np.float32)
    self.data_array = self.data_array * self.scale
    self.data_array = self.data_array.view(np.complex64)
    return self.data_array

read_header()

Parses text header part.

Returns:

Name Type Description
dictionary

Dictionary of values
Source code in iqtools/tcapdata.py 
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
def read_header(self):
    """Parses text header part.

    Returns:
        dictionary: Dictionary of values
    """        
    dic = {}
    with open(self.header_filename) as f:
        for line in f:
            name, var = line.split()
            dic[name.strip()] = var
    self.version = dic['version']
    self.center = float(dic['center_freq'])
    self.adc_range = float(dic['adc_range'])
    self.scale = float(dic['data_scale'])
    self.block_count = int(dic['block_count'])
    self.block_size = int(dic['block_size'])
    self.frame_size = int(dic['frame_size'])
    self.decimation = int(dic['decimation'])
    self.trigger_time = float(dic['trigger_time'])
    self.segment_blocks = int(dic['segment_blocks'])
    return dic

read_samples(nsamples, offset=0)

Reads TCAP data.

TCAP format information:

- Each file contains 15625 blocks equivalent of 27min 18 seconds of recording
- Each block is 2^17=131072 BYTES of data + 88 bytes of header
- Each sample is 4 bytes = 32bits (2 I + 2 Q bytes), hence each block contains
32768 complex valued samples
- Sampling frequency is 312500 sps, thus the data of a block are each 0.105s
worth of data and a resolution frequency of 312500 / 32768 = 9.5 Hz per block
- To double the frequency resolution one can take two consecutive blocks which
mean 4.77 Hz for two consecutive blocks which is 0.210 s of time.
- Either from one block or two blocks a frame can be created.
- An FFT is done on each frame. 10 such FFTs can be averaged to reduce noise.
e.g. a 2-block frame would correspond to 2.12s data.
- 400 such averaged groups of 10 frames can be plotted on a single spectrogram
- each picture of a single spectrogram would then correspond to 14m and 8 sec.

Parameters:

Name Type Description Default
nsamples int

Number of samples to read

 required
offset int

Starting sample. Defaults to 0.

 0
Source code in iqtools/tcapdata.py 
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
def read_samples(self, nsamples, offset=0):
    """Reads TCAP data.

    TCAP format information:

        - Each file contains 15625 blocks equivalent of 27min 18 seconds of recording
        - Each block is 2^17=131072 BYTES of data + 88 bytes of header
        - Each sample is 4 bytes = 32bits (2 I + 2 Q bytes), hence each block contains
        32768 complex valued samples
        - Sampling frequency is 312500 sps, thus the data of a block are each 0.105s
        worth of data and a resolution frequency of 312500 / 32768 = 9.5 Hz per block
        - To double the frequency resolution one can take two consecutive blocks which
        mean 4.77 Hz for two consecutive blocks which is 0.210 s of time.
        - Either from one block or two blocks a frame can be created.
        - An FFT is done on each frame. 10 such FFTs can be averaged to reduce noise.
        e.g. a 2-block frame would correspond to 2.12s data.
        - 400 such averaged groups of 10 frames can be plotted on a single spectrogram
        - each picture of a single spectrogram would then correspond to 14m and 8 sec.

    Args:
        nsamples (int): Number of samples to read
        offset (int, optional): Starting sample. Defaults to 0.
    """        

    BLOCK_HEADER_SIZE = 88
    BLOCK_DATA_SIZE = 2 ** 17
    BLOCK_SIZE = BLOCK_HEADER_SIZE + BLOCK_DATA_SIZE

    filesize = os.path.getsize(self.filename)
    # each file contains 15625 blocks
    if not filesize == 15625 * BLOCK_SIZE:
        log.info(
            "File size does not match block sizes times total number of blocks. Aborting...")
        return

    # read header section
    with open(self.filename, 'rb') as f:
        tfp = f.read(12)
        pio = f.read(12)
        scalers = f.read(64)

    self.date_time = self.parse_tcap_tfp(tfp)
    self.tcap_pio = pio
    self.tcap_scalers = scalers

    data_section_size = self.frame_size - BLOCK_HEADER_SIZE
    n_iq_samples = data_section_size / 2 / 2  # two bytes for I and two bytes for Q
    self.nsamples_total = self.segment_blocks * n_iq_samples

    # 4 comes from 2 times 2 byte integer for I and Q
    total_n_bytes = 4 * nsamples
    start_n_bytes = 4 * offset

    ba = bytearray()
    try:
        with open(self.filename, 'rb') as f:
            f.seek(BLOCK_HEADER_SIZE + start_n_bytes)
            for i in range(total_n_bytes):
                if not f.tell() % BLOCK_SIZE:
                    log.info(
                        'File pointer before jump: {}'.format(f.tell()))
                    log.info(
                        "Reached end of block {}. Now skipoing header of block {}!".format(
                            int(f.tell() / BLOCK_SIZE),
                            int(
                                f.tell() / BLOCK_SIZE) + 1))
                    f.seek(88, 1)
                    log.info('File pointer after jump: {}'.format(f.tell()))
                # using bytearray.extend is much faster than using +=
                ba.extend(f.read(1))
    except:
        log.error('File seems to end here!')
        return

    log.info('Total bytes read: {}'.format(len(ba)))

    # big endian 16 bit for I and 16 bit for Q
    self.data_array = np.frombuffer(ba, '>i2')
    self.data_array = self.data_array.astype(np.float32)
    self.data_array = self.data_array * self.scale
    self.data_array = self.data_array.view(np.complex64)