int Compress	(	const string &	ifile,
		const string &	ofile
	)

Definition at line 66 of file zfits.cc.

References Huffman::Encode(), fits::GetColumns(), fits::GetNextRow(), fits::GetNumRows(), fits::GetRow(), if(), fits::SetPtrAddress(), start(), and Time::UnixTime().

Referenced by main().

{
    // when to print some info on the screen (every f percent)
    float frac = 0.01;

    // open a fits file
    fits f(ifile);

    // open output file
    ofstream fout(ofile);

    // counters for total size and compressed size
    uint64_t tot = 0;
    uint64_t com = 0;

    // very simple timer
    double sec = 0;

    // Produce a lookup table with all informations about the
    // columns in the same order as they are in the file
    const fits::Table::Columns &cols= f.GetColumns();

    struct col_t : fits::Table::Column
    {
        string name;
        void *ptr;
    };


    map<size_t, col_t> columns;

    size_t row_tot = 0;
    for (auto it=cols.begin(); it!=cols.end(); it++)
    {
        col_t c;

        c.offset = it->second.offset;
        c.size   = it->second.size;
        c.num    = it->second.num;
        c.name   = it->first;
        c.ptr    = f.SetPtrAddress(it->first);

        columns[c.offset] = c;

        row_tot += c.size*c.num;
    }

    // copy the header from the input to the output file
    // and prefix the output file as a compressed fits file
    string header;
    header.resize(f.tellg());

    f.seekg(0);
    f.read((char*)header.c_str(), header.size());

    char m[2];
    m[0] = 'z'+128;
    m[1] = 'f'+128;

    const size_t hlen = 0;

    size_t hs = header.size();

    fout.write(m, 2);                           // magic number
    fout.write((char*)&hlen, sizeof(size_t));   // length of possible header data (e.g. file version, compression algorithm)
    fout.write((char*)&hs,   sizeof(size_t));   // size of FITS header
    fout.write(header.c_str(), header.size());  // uncompressed FITS header

    tot += header.size();
    com += header.size()+2+2*sizeof(size_t);

    cout << fixed;

    Time start;

    // loop over all rows
    vector<char> cache(row_tot);
    while (f.GetNextRow())
    {
        // pointer to the start of the cache for the data of one row
        char *out = cache.data();

        // mask stroing which column have been compressed and which not
        vector<uint8_t> mask(cols.size()/8 + 1);

        // loop over all columns
        uint32_t icol = 0;
        for (auto it=columns.begin(); it!=columns.end(); it++, icol++)
        {
            // size of cell in bytes
            const size_t len_col = it->second.size * it->second.num;

            // get pointer to data
            int16_t *ptr = (int16_t*)it->second.ptr;

            // If the column is the data, preprocess the data
            /*
            if (it->second.name=="Data")
            {
                int16_t *end = ptr+1440*300-4-(1440*300)%2;
                int16_t *beg = ptr;

                while (end>=beg)
                {
                    const int16_t avg = (end[0] + end[1])/2;
                    end[2] -= avg;
                    end[3] -= avg;
                    end -=2;
                }
            }*/

            // do not try to compress less than 32bytes
            if (len_col>32 && it->second.size==2)
            {
                Time now;

                // perform 16bit hoffman (option for 8bit missing, skip 64bit)
                // (what to do with floats?)
                string buf;
                /*int len =*/ Huffman::Encode(buf, (uint16_t*)ptr, len_col/2);

                sec += Time().UnixTime()-now.UnixTime();

                // check if data was really compressed
                if (buf.size()<len_col)
                {
                    // copy compressed data into output cache
                    memcpy(out, buf.c_str(), buf.size());
                    out += buf.size();

                    // update mask
                    const uint64_t bit = (icol%8);
                    mask[icol/8] |= (1<<bit);

                    continue;
                }
            }

            // just copy the data if it has not been compressed
            memcpy(out, (char*)ptr, len_col);
            out += len_col;
        }

        // calcualte size of output buffer
        const size_t sz = out-cache.data();

        // update counters
        tot += row_tot;
        com += sz + mask.size();

        // write the compression mask and the (partly) copmpressed data stream
        fout.write((char*)mask.data(), mask.size());
        fout.write(cache.data(), sz);

         //if (sz2<0 || memcmp(data, dest3.data(), 432000*2)!=0)
         //   cout << "grrrr" << endl;

        const float proc = float(f.GetRow())/f.GetNumRows();
        if (proc>frac)
        {
            const double elep = Time().UnixTime()-start.UnixTime();
            cout << "\r" << setprecision(0) << setw(3) << 100*proc << "% [" << setprecision(1) << setw(5) << 100.*com/tot << "%] cpu:" << sec << "s in:" << tot/1000000/elep << "MB/s" << flush;
            frac += 0.01;
        }
    }

    const double elep = Time().UnixTime()-start.UnixTime();
    cout << setprecision(0) << "\r100% [" << setprecision(1) << setw(5) << 100.*com/tot << "%] cpu:"  << sec << "s in:" << tot/1000000/elep << "MB/s" << endl;

    return 0;
}

Here is the call graph for this function:

Here is the caller graph for this function: