Row-major Encoding¶

Shown below is a simple program that encodes sample data from a standardised row-major memory layout and outputs an ODB-2 file.

Note

The C++ interface does not support automatically encoding data from a row-major layout. If this functionality is needed, the C API should be used. Alternatively the row-major memory layout can be explicitly specified as a custom memory layout, which is what the C API does internally.

/**
 * To build this program, please make sure to reference linked library:
 *
 *     gcc -lodccore -o odc-c-encode-row-major odc_encode_row_major.c
 */

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>

#include "odc/api/odc.h"

// Bitfield constants
#define Ob00000001 1
#define Ob00001011 11
#define Ob01101011 107

#define CHECK_RESULT(x) \
    do { \
        int rc = (x); \
        if (rc != ODC_SUCCESS) { \
            fprintf(stderr, "Error calling odc function \"%s\": %s\n", #x, odc_error_string(rc)); \
            exit(1); \
        } \
    } while (false); \

void usage() {
    fprintf(stderr, "Usage:\n");
    fprintf(stderr, "    odc-c-encode-row-major <odb2 output file>\n\n");
}

void cycle_longs(long *list, int size, long *pool, int pool_size) {
    int index = 0;
    int i;

    for (i = 0; i < size; i++) {
        if (index == pool_size) index = 0;
        list[i] = pool[index];
        index++;
    }
}

void cycle_doubles(double *list, int size, double *pool, int pool_size) {
    int index = 0;
    int i;

    for (i = 0; i < size; i++) {
        if (index == pool_size) index = 0;
        list[i] = pool[index];
        index++;
    }
}

void create_scratch_data(long nrows, int ncols, double data[nrows][ncols]) {

    // Prepare the current date as an integer

    time_t rawtime;
    time(&rawtime);

    struct tm * timeinfo;
    timeinfo = localtime(&rawtime);

    int64_t date = 10000 * (timeinfo->tm_year + 1900) + 100 * (timeinfo->tm_mon + 1) + timeinfo->tm_mday;

    // Prepare the list of integer values, including the missing value

    long missing_integer;
    CHECK_RESULT(odc_missing_integer(&missing_integer));

    long integer_pool[] = { 1234, 4321, missing_integer };
    int integer_pool_size = sizeof(integer_pool)/sizeof(integer_pool[0]);

    long missing_integers[nrows];
    cycle_longs(missing_integers, nrows, integer_pool, integer_pool_size);

    // Prepare the list of double values, including the missing value

    double missing_double;
    CHECK_RESULT(odc_missing_double(&missing_double));

    double double_pool[] = { 12.34, 43.21, missing_double };
    int double_pool_size = sizeof(double_pool)/sizeof(double_pool[0]);

    double missing_doubles[nrows];
    cycle_doubles(missing_doubles, nrows, double_pool, double_pool_size);

    // Prepare the list of bitfield values

    long bitfield_pool[] = { Ob00000001, Ob00001011, Ob01101011 };
    int bitfield_pool_size = sizeof(bitfield_pool)/sizeof(bitfield_pool[0]);

    long bitfield_values[nrows];
    cycle_longs(bitfield_values, nrows, bitfield_pool, bitfield_pool_size);

    int i;

    // Fill in the passed data array with scratch values
    for (i = 0; i < nrows; i++) {
        memset(&data[i][0], 0, 5);
        snprintf((char*)(&data[i][0]), 5, "xxxx");  // expver

        *(int64_t*)(&data[i][1]) = date;  // date@hdr

        memset(&data[i][2], 0, 8);
        snprintf((char*)(&data[i][2]), 7, "stat%02d", i);  // statid@hdr

        memset(&data[i][3], 0, 16);
        snprintf((char*)(&data[i][3]), 16, "0-12345-0-678%02d", i);  // wigos@hdr

        *(double*)(&data[i][5]) = 12.3456 * i;  // obsvalue@body
        *(long*)(&data[i][6]) = missing_integers[i];  // integer_missing
        *(double*)(&data[i][7]) = missing_doubles[i];  // double_missing
        *(long*)(&data[i][8]) = bitfield_values[i];  // bitfield_column
    }
}

int main(int argc, char* argv[]) {
    if (argc != 2) {
        usage();
        return 1;
    }

    // Get output path from command argument
    char* path = argv[1];

    // Initialise API and set treatment of integers as longs
    CHECK_RESULT(odc_initialise_api());
    CHECK_RESULT(odc_integer_behaviour(ODC_INTEGERS_AS_LONGS));

    // Define row and column count
    const long nrows = 20;
    const int ncols = 9;

    // Allocate data array of rows x columns
    double data[nrows][ncols];

    // Set up the allocated array with scratch data
    create_scratch_data(nrows, ncols, data);

    // Initialise encoder
    odc_encoder_t* encoder = NULL;
    CHECK_RESULT(odc_new_encoder(&encoder));

    // Define all column names and their types
    CHECK_RESULT(odc_encoder_add_column(encoder, "expver", ODC_STRING));
    CHECK_RESULT(odc_encoder_add_column(encoder, "date@hdr", ODC_INTEGER));
    CHECK_RESULT(odc_encoder_add_column(encoder, "statid@hdr", ODC_STRING));
    CHECK_RESULT(odc_encoder_add_column(encoder, "wigos@hdr", ODC_STRING));
    CHECK_RESULT(odc_encoder_add_column(encoder, "obsvalue@body", ODC_REAL));
    CHECK_RESULT(odc_encoder_add_column(encoder, "integer_missing", ODC_INTEGER));
    CHECK_RESULT(odc_encoder_add_column(encoder, "double_missing", ODC_REAL));
    CHECK_RESULT(odc_encoder_add_column(encoder, "bitfield_column", ODC_BITFIELD));

    // Column `wigos@hdr` is a 16-byte string column, hence takes 2 columns in the array => ncols=9
    CHECK_RESULT(odc_encoder_column_set_data_size(encoder, 3, 16));

    // Column `bitfield_column` is an integer with 4 bitfield values in it
    CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_a", 1));
    CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_b", 2));
    CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_c", 3));
    CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_d", 1));

    // Set input data array from which data will be encoded
    //   Zero value for last `columnMajorWidth` argument indicates the row-major layout
    CHECK_RESULT(odc_encoder_set_data_array(encoder, data, ncols*sizeof(double), nrows, 0));

    const char* property_key = "encoded_by";
    const char* property_value = "odc_example";

    // Add some key/value metadata to the frame
    CHECK_RESULT(odc_encoder_add_property(encoder, property_key, property_value));

    int file_descriptor = open(path, O_CREAT|O_TRUNC|O_WRONLY, 0666);
    long size;

    // Encode ODB-2 into an already open file descriptor
    CHECK_RESULT(odc_encode_to_file_descriptor(encoder, file_descriptor, &size));

    close(file_descriptor);

    // Deallocate memory used up by the encoder
    CHECK_RESULT(odc_free_encoder(encoder));

    fprintf(stdout, "Written %ld rows to %s\n", nrows, path);

    return 0;
}

To use this sample program, invoke it from the command line with a path to an ODB-2 output file:

./odc-c-encode-row-major example.odb

Written 20 rows to example.odb

odc ls example.odb

expver date@hdr        statid@hdr      wigos@hdr               obsvalue@body   integer_missing double_missing  bitfield_column
'xxxx' 20210524        'stat00'        '0-12345-0-67800'       0.000000        1234            12.340000       1
'xxxx' 20210524        'stat01'        '0-12345-0-67801'       12.345600       4321            43.209999       11
'xxxx' 20210524        'stat02'        '0-12345-0-67802'       24.691200       .               .               107
'xxxx' 20210524        'stat03'        '0-12345-0-67803'       37.036800       1234            12.340000       1
'xxxx' 20210524        'stat04'        '0-12345-0-67804'       49.382401       4321            43.209999       11
'xxxx' 20210524        'stat05'        '0-12345-0-67805'       61.728001       .               .               107
'xxxx' 20210524        'stat06'        '0-12345-0-67806'       74.073601       1234            12.340000       1
'xxxx' 20210524        'stat07'        '0-12345-0-67807'       86.419197       4321            43.209999       11
'xxxx' 20210524        'stat08'        '0-12345-0-67808'       98.764801       .               .               107
'xxxx' 20210524        'stat09'        '0-12345-0-67809'       111.110397      1234            12.340000       1
'xxxx' 20210524        'stat10'        '0-12345-0-67810'       123.456001      4321            43.209999       11
'xxxx' 20210524        'stat11'        '0-12345-0-67811'       135.801605      .               .               107
'xxxx' 20210524        'stat12'        '0-12345-0-67812'       148.147202      1234            12.340000       1
'xxxx' 20210524        'stat13'        '0-12345-0-67813'       160.492798      4321            43.209999       11
'xxxx' 20210524        'stat14'        '0-12345-0-67814'       172.838394      .               .               107
'xxxx' 20210524        'stat15'        '0-12345-0-67815'       185.184006      1234            12.340000       1
'xxxx' 20210524        'stat16'        '0-12345-0-67816'       197.529602      4321            43.209999       11
'xxxx' 20210524        'stat17'        '0-12345-0-67817'       209.875198      .               .               107
'xxxx' 20210524        'stat18'        '0-12345-0-67818'       222.220795      1234            12.340000       1
'xxxx' 20210524        'stat19'        '0-12345-0-67819'       234.566406      4321            43.209999       11
000 2021-05-24 14:06:05 (I) Selected 20 row(s).

! To build this program, please make sure to first compile odc Fortran module,
! and then reference linked libraries:
!
!     gfortran -c ../../src/odc/api/odc.f90
!     gfortran -lodccore -lfodc -o odc-fortran-encode-row-major odc_encode_row_major.f90

program odc_ls
    use, intrinsic :: iso_c_binding, only: c_null_char
    use odc
    implicit none

    character(255) :: path

    integer, parameter :: ncols = 9
    integer(8), parameter :: nrows = 20
    real(8), target :: data(ncols, nrows)

    type(odc_encoder) :: encoder
    logical, parameter :: column_major = .false.
    character(10), parameter :: property_key = 'encoded_by'
    character(11), parameter :: property_value = 'odc_example'
    integer, target :: outunit
    integer(8), target :: bytes_written

    if (command_argument_count() /= 1) then
        call usage()
        stop 1
    end if

    ! Get output path from command argument
    call get_command_argument(1, path)

    ! Initialise API and set treatment of integers as longs
    call check_call(odc_initialise_api(), 'initialising api')
    call check_call(odc_integer_behaviour(ODC_INTEGERS_AS_LONGS), 'setting integer behaviour to longs')

    ! Set up the allocated array with scratch data
    call create_scratch_data(data, nrows)

    ! Initialise encoder
    call check_call(encoder%initialise(), 'initialising encoder')

    ! Define all column names and their types
    call check_call(encoder%add_column('expver', ODC_STRING), 'adding expver column')
    call check_call(encoder%add_column('date@hdr', ODC_INTEGER), 'adding date@hdr column')
    call check_call(encoder%add_column('statid@hdr', ODC_STRING), 'adding statid@hdr column')
    call check_call(encoder%add_column('wigos@hdr', ODC_STRING), 'adding wigos@hdr column')
    call check_call(encoder%add_column('obsvalue@body', ODC_REAL), 'adding obsvalue@bod column')
    call check_call(encoder%add_column('integer_missing', ODC_INTEGER), 'adding integer_missing column')
    call check_call(encoder%add_column('double_missing', ODC_REAL), 'adding double_missing column')
    call check_call(encoder%add_column('bitfield_column', ODC_BITFIELD), 'adding bitfield_column')

    ! Column `wigos@hdr` is a 16-byte string column, hence takes 2 columns in the array => ncols=9
    call check_call(encoder%column_set_data_size(4, element_size_doubles=2), 'setting column data size')

    ! Column `bitfield_column` is an integer with 4 bitfield values in it
    call check_call(encoder%column_add_bitfield(8, "flag_a", 1), 'adding flag_a bitfield')
    call check_call(encoder%column_add_bitfield(8, "flag_b", 2), 'adding flag_b bitfield')
    call check_call(encoder%column_add_bitfield(8, "flag_c", 3), 'adding flag_c bitfield')
    call check_call(encoder%column_add_bitfield(8, "flag_d", 1), 'adding flag_d bitfield')

    ! Set input data array from which data will be encoded
    call check_call(encoder%set_data(data, column_major), 'setting encoder data array')

    ! Add some key/value metadata to the frame
    call check_call(encoder%add_property(property_key, property_value), 'adding property')

    ! Encode ODB-2 into a file
    open(newunit=outunit, file=path, access='stream', form='unformatted', status='replace')
    call check_call(encoder%encode(outunit, bytes_written), 'encoding data')
    close(outunit)

    ! Deallocate memory used up by the encoder
    call check_call(encoder%free(), 'cleaning up encoder')

    write(6, '(a,i0,a,a)') 'Written ', nrows, ' rows to ', trim(path)

contains

    subroutine check_call(err, desc)
        integer, intent(in) :: err
        character(*), intent(in) :: desc

        if (err /= ODC_SUCCESS) then
            write(7, *) '**** An error occurred in ODC library'
            write(7, *) 'Description: ', desc
            write(7, *) 'Error: ', odc_error_string(err)
            stop 1
        end if
    end subroutine

    subroutine usage()
        write(6, *) 'Usage:'
        write(6, *) '    odc-fortran-encode-row-major <odb2 output file>'
    end subroutine

    subroutine cycle_ints(output, input)
        integer(8), intent(in) :: input(:)
        integer(8), intent(out) :: output(:)
        integer :: i

        do i = 1, size(output, 1)
            output(i) = input(mod(i - 1, size(input, 1)) + 1)
        end do
    end subroutine

    subroutine cycle_reals(output, input)
        real(8), intent(in) :: input(:)
        real(8), intent(out) :: output(:)
        integer :: i

        do i = 1, size(output, 1)
            output(i) = input(mod(i - 1, size(input, 1)) + 1)
        end do
    end subroutine

    subroutine create_scratch_data(data, nrows)
        real(8), intent(out) :: data(:,:)
        integer(8), intent(in) :: nrows

        character(5) :: expver_str
        character(8) :: date_str
        integer(8) :: date
        character(7) :: statid_str
        character(16) :: wigos_str
        integer(8) :: missing_integer
        integer(8) :: integer_pool(3)
        integer(8), target :: missing_integers(nrows)
        real(8) :: missing_double
        real(8) :: double_pool(3)
        real(8), target :: missing_doubles(nrows)
        integer(8) :: bitfield_pool(3)
        integer(8), target :: bitfield_values(nrows)
        integer(8) :: i

        ! Prepare the current date as an integer
        call date_and_time(date=date_str)
        read(date_str, *) date

        ! Prepare the list of integer values, including the missing value
        call check_call(odc_missing_integer(missing_integer), 'getting missing integer value')
        integer_pool = (/ int(1234, 8), int(4321, 8), missing_integer /)
        call cycle_ints(missing_integers, integer_pool)

        ! Prepare the list of double values, including the missing value
        call check_call(odc_missing_double(missing_double), 'getting missing double value')
        double_pool = (/ real(12.34, 8), real(43.21, 8), missing_double /)
        call cycle_reals(missing_doubles, double_pool)

        ! Prepare the list of bitfield values
        bitfield_pool = (/ int(b'00000001'), int(b'00001011'), int(b'01101011') /)
        call cycle_ints(bitfield_values, bitfield_pool)

        ! Fill in the passed data array with scratch values
        do i = 1, nrows
            write(expver_str, '(a,a)') 'xxxx', '' // c_null_char  ! proper string termination
            data(1, i) = transfer(expver_str, data(1, i))  ! expver

            data(2, i) = transfer(date, data(2, i))  ! date@hdr

            write(statid_str, '(a,i0.2,a)') 'stat', i - 1, '' // c_null_char  ! proper string termination
            data(3, i) = transfer(statid_str, data(3, i))  ! statid@hdr

            write(wigos_str, '(a,i0.2,a)') '0-12345-0-678', i - 1, '' // c_null_char  ! proper string termination
            data(4:5, i) = transfer(wigos_str, data(4:5, i))  ! wigos@hdr

            data(6, i) = 12.3456 * real(i - 1)  ! obsvalue@body
            data(7, i) = transfer(missing_integers(i), data(7, i))  ! integer_missing
            data(8, i) = missing_doubles(i)  ! double_missing
            data(9, i) = transfer(bitfield_values(i), data(9, i))  ! bitfield_column
        end do
    end subroutine

end program

To use this sample program, invoke it from the command line with a path to an ODB-2 output file:

./odc-fortran-encode-row-major example.odb

Written 20 rows to example.odb

odc ls example.odb

expver date@hdr        statid@hdr      wigos@hdr               obsvalue@body   integer_missing double_missing  bitfield_column
'xxxx' 20210524        'stat00'        '0-12345-0-67800'       0.000000        1234            12.340000       1
'xxxx' 20210524        'stat01'        '0-12345-0-67801'       12.345600       4321            43.209999       11
'xxxx' 20210524        'stat02'        '0-12345-0-67802'       24.691200       .               .               107
'xxxx' 20210524        'stat03'        '0-12345-0-67803'       37.036800       1234            12.340000       1
'xxxx' 20210524        'stat04'        '0-12345-0-67804'       49.382401       4321            43.209999       11
'xxxx' 20210524        'stat05'        '0-12345-0-67805'       61.728001       .               .               107
'xxxx' 20210524        'stat06'        '0-12345-0-67806'       74.073601       1234            12.340000       1
'xxxx' 20210524        'stat07'        '0-12345-0-67807'       86.419205       4321            43.209999       11
'xxxx' 20210524        'stat08'        '0-12345-0-67808'       98.764801       .               .               107
'xxxx' 20210524        'stat09'        '0-12345-0-67809'       111.110397      1234            12.340000       1
'xxxx' 20210524        'stat10'        '0-12345-0-67810'       123.456001      4321            43.209999       11
'xxxx' 20210524        'stat11'        '0-12345-0-67811'       135.801605      .               .               107
'xxxx' 20210524        'stat12'        '0-12345-0-67812'       148.147202      1234            12.340000       1
'xxxx' 20210524        'stat13'        '0-12345-0-67813'       160.492798      4321            43.209999       11
'xxxx' 20210524        'stat14'        '0-12345-0-67814'       172.838409      .               .               107
'xxxx' 20210524        'stat15'        '0-12345-0-67815'       185.184006      1234            12.340000       1
'xxxx' 20210524        'stat16'        '0-12345-0-67816'       197.529602      4321            43.209999       11
'xxxx' 20210524        'stat17'        '0-12345-0-67817'       209.875198      .               .               107
'xxxx' 20210524        'stat18'        '0-12345-0-67818'       222.220795      1234            12.340000       1
'xxxx' 20210524        'stat19'        '0-12345-0-67819'       234.566406      4321            43.209999       11
000 2021-05-24 14:38:35 (I) Selected 20 row(s).