Skip to content

stream: improved support for large tag files #921

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ourairquality wants to merge 1 commit into
base: main
Choose a base branch
from
+65 −35
Open

stream: improved support for large tag files #921

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 65 additions & 35 deletions src/stream.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@
* use integer types in stdint.h
*-----------------------------------------------------------------------------*/
#define _POSIX_C_SOURCE 200112L

#ifndef WIN32
// For 64 bit file offsets on Linux and MacOS.
#define _FILE_OFFSET_BITS 64
#endif

#include <ctype.h>
#include "rtklib.h"
#ifndef WIN32
Expand Down Expand Up @@ -155,7 +161,11 @@ typedef struct { /* file control type */
gtime_t wtime; /* write time */
uint32_t tick; /* start tick */
uint32_t tick_f; /* start tick in file */
long fpos_n; /* next file position */
#ifdef _WIN32
int64 fpos_n; // Next file position
#else
off_t fpos_n; // Next file position
#endif
uint32_t tick_n; /* next tick */
double start; /* start offset (s) */
double speed; /* replay speed (time factor) */
Expand Down Expand Up @@ -818,10 +828,7 @@ static int readfile(file_t *file, uint8_t *buff, int nmax, char *msg)
{
struct timeval tv={0};
fd_set rs;
uint64_t fpos_8B;
uint32_t t,tick,fpos_4B;
long pos,n;
int nr=0;
uint32_t t,tick;

tracet(4,"readfile: fp=%d nmax=%d\n",file->fp,nmax);

Expand All @@ -832,8 +839,9 @@ static int readfile(file_t *file, uint8_t *buff, int nmax, char *msg)
/* input from stdin */
FD_ZERO(&rs); FD_SET(0,&rs);
if (!select(1,&rs,NULL,NULL,&tv)) return 0;
if ((nr=(int)read(0,buff,nmax))<0) return 0;
return nr;
ssize_t nr = read(0, buff, nmax);
if (nr < 0) return 0;
return (int)nr;
#else
return 0;
#endif
Expand All @@ -850,18 +858,30 @@ static int readfile(file_t *file, uint8_t *buff, int nmax, char *msg)
}
/* seek time-tag file to get next tick and file position */
while ((int)(file->tick_n-t)<=0) {

uint32_t fpos_4B;
uint64_t fpos_8B;
if (fread(&file->tick_n,sizeof(tick),1,file->fp_tag)<1||
fread((file->size_fpos==4)?(void *)&fpos_4B:(void *)&fpos_8B,
file->size_fpos,1,file->fp_tag)<1) {
file->tick_n=(uint32_t)(-1);
pos=ftell(file->fp);
fseek(file->fp,0L,SEEK_END);
file->fpos_n=ftell(file->fp);
fseek(file->fp,pos,SEEK_SET);
#ifdef _WIN32
__int64 pos = _ftelli64(file->fp);
_fseeki64(file->fp, 0, SEEK_END);
file->fpos_n = _ftelli64(file->fp);
_fseeki64(file->fp, pos, SEEK_SET);
#else
off_t pos = ftello(file->fp);
fseeko(file->fp, 0, SEEK_END);
file->fpos_n = ftello(file->fp);
fseeko(file->fp, pos, SEEK_SET);
#endif
break;
}
file->fpos_n=(long)((file->size_fpos==4)?fpos_4B:fpos_8B);
#ifdef _WIN32
file->fpos_n = (int64_t)((file->size_fpos == 4) ? fpos_4B : fpos_8B);
#else
file->fpos_n = (off_t)((file->size_fpos == 4) ? fpos_4B : fpos_8B);
#endif
}
if (file->tick_n==(uint32_t)(-1)) {
sprintf(msg,"end");
Expand All @@ -871,29 +891,29 @@ static int readfile(file_t *file, uint8_t *buff, int nmax, char *msg)
file->wtime=timeadd(file->time,(int)t*0.001);
timeset(timeadd(gpst2utc(file->time),(int)file->tick_n*0.001));
}
if ((n=file->fpos_n-ftell(file->fp))<nmax) {
nmax=n;
}
#ifdef _WIN32
int64_t n = file->fpos_n - _ftelli64(file->fp);
#else
off_t n = file->fpos_n - ftello(file->fp);
#endif
if (n < nmax) nmax = (int)n;
}
size_t nr = 0;
if (nmax>0) {
nr=(int)fread(buff,1,nmax,file->fp);
}
if (feof(file->fp)) {
sprintf(msg,"end");
nr = fread(buff,1,nmax,file->fp);
}
tracet(5,"readfile: fp=%d nr=%d\n",file->fp,nr);
return nr;
if (feof(file->fp)) sprintf(msg,"end");
tracet(5,"readfile: fp=%d nr=%zd\n",file->fp,nr);
return (int)nr;
}
/* write file ----------------------------------------------------------------*/
static int writefile(file_t *file, uint8_t *buff, int n, char *msg)
{
gtime_t wtime;
uint64_t fpos_8B;
uint32_t tick=tickget(),fpos_4B;
int week1,week2,ns;
uint32_t tick=tickget();
int week1,week2;
double tow1,tow2,intv;
long fpos,fpos_tmp=0;


tracet(4,"writefile: fp=%d n=%d\n",file->fp,n);

if (!file) return 0;
Expand All @@ -918,43 +938,53 @@ static int writefile(file_t *file, uint8_t *buff, int n, char *msg)
}
if (!file->fp) return 0;

ns=(int)fwrite(buff,1,n,file->fp);
fpos=ftell(file->fp);
size_t ns = fwrite(buff, 1, n, file->fp);
#ifdef _WIN32
int64_t fpos = _ftelli64(file->fp);
int64_t fpos_tmp = 0;
#else
off_t fpos = ftello(file->fp);
off_t fpos_tmp = 0;
#endif
fflush(file->fp);
file->wtime=wtime;

if (file->fp_tmp) {
fwrite(buff,1,n,file->fp_tmp);
fpos_tmp=ftell(file->fp_tmp);
#ifdef _WIN32
fpos_tmp = _ftelli64(file->fp_tmp);
#else
fpos_tmp = ftello(file->fp_tmp);
#endif
fflush(file->fp_tmp);
}
if (file->fp_tag) {
tick-=file->tick;
fwrite(&tick,1,sizeof(tick),file->fp_tag);
if (file->size_fpos==4) {
fpos_4B=(uint32_t)fpos;
uint32_t fpos_4B = (uint32_t)fpos;
fwrite(&fpos_4B,1,sizeof(fpos_4B),file->fp_tag);
}
else {
fpos_8B=(uint64_t)fpos;
uint64_t fpos_8B = (uint64_t)fpos;
fwrite(&fpos_8B,1,sizeof(fpos_8B),file->fp_tag);
}
fflush(file->fp_tag);

if (file->fp_tag_tmp) {
fwrite(&tick,1,sizeof(tick),file->fp_tag_tmp);
if (file->size_fpos==4) {
fpos_4B=(uint32_t)fpos_tmp;
uint32_t fpos_4B = (uint32_t)fpos_tmp;
fwrite(&fpos_4B,1,sizeof(fpos_4B),file->fp_tag_tmp);
}
else {
fpos_8B=(uint64_t)fpos_tmp;
uint64_t fpos_8B = (uint64_t)fpos_tmp;
fwrite(&fpos_8B,1,sizeof(fpos_8B),file->fp_tag_tmp);
}
fflush(file->fp_tag_tmp);
}
}
tracet(5,"writefile: fp=%d ns=%d tick=%5d fpos=%d\n",file->fp,ns,tick,fpos);
tracet(5,"writefile: fp=%d ns=%d tick=%5d fpos=%ld\n",file->fp,ns,tick,(long long)fpos);

return ns;
}
Expand Down