Linux command to retrieve a byte range from a file

Question

I know that head and tail can take -c option to specify a byte offset. I'm looking for a way to efficiently extract a byte range from a large log file.

Answer 1

The DareDevil of the Unix commands, dd to the rescue!

dd if=yourfile ibs=1 skip=200 count=100

That would start from byte 200 and show 100 next bytes, or in other words, bytes 200-300. ibs means dd only reads one byte at a time instead of the default 512 bytes, but still writes out in default 512 byte chunks. Go and see if ibs harms the performance, I hope not.

Answer 2

If your interest is in the bytes, would od be of more interest.

-j, --skip-bytes=bytes
-N, --read-bytes=bytes

So to read the 16 bytes starting at byte 1024, and output in ascii

od -j 1024 -N 16 -a /bin/sh

Answer 3

You can use dd if=logfile of=pieceoflogfile skip=startingblock count=#ofblocks (possibly with bs=1 to get one-byte blocks, otherwise it uses 512 byte blocks). Not sure how efficient it is to tell it to write one byte at a time, though.

Answer 4

Assuming the file isn't excessively large (e.g. several GB or so), piping from one to the next is as efficient as you'll get, short of writing your own program to do so.

head ... file | tail ...

(Or the other way around. Whichever.)

Answer 5

I know this is old, and asked for a "linux" command in the title, but python is perfect for this and comes with most linux distributions. It's as easy as this:

python -c 'f=open("myfile.txt","rb");f.seek(100);print(f.read(100));f.close()' | SomeOtherProgram

f.seek(n,0) goes n bytes from the start of the program, and f.read(n) reads the number of bytes you want.

I've not found any other options that make it that easy.

Answer 6

u can use splice in linux, this example in c program, and then name it fastcat.c

#define _GNU_SOURCE
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>

#define __IO_BUFSIZE (8192)

int main(int argc, char *argv[])
{
loff_t in_off = 0;
loff_t out_off = 0;
off_t lenData;
int StartReadDataAt=-1;
int AmountOfData=-1;
char location[100];
int err = -1;
static int buf_size = __IO_BUFSIZE;
struct stat stbuf;

for (int i = 1; i < argc; ++i)
{
    if (argv[i][0] == '-' && argv[i][1] == 'f') //argument for StartReadDataAt
    {
    int totalIndeks =strlen(argv[i])-3;
    char data[totalIndeks];

    strncpy(data, argv[i] + 3,totalIndeks);
    data[totalIndeks]='\0';
    StartReadDataAt=atoi(data);

    StartReadDataAt=StartReadDataAt-1;
    if(StartReadDataAt == -1){
        StartReadDataAt=0;
        }
    }
    else if (argv[i][0] == '-' && argv[i][1] == 'c') //argument for AmountOfData
    {
    int totalIndeks =strlen(argv[i])-3;
        char data[totalIndeks];

        strncpy(data, argv[i] + 3, totalIndeks );
    data[totalIndeks]='\0';
        AmountOfData=atoi(data);
    }
    else                        //argument for Location
    {
        strncpy(location, argv[i], strlen(argv[i]));
    location[strlen(argv[i])]='\0';
    }
}


    FILE* f = fopen(location, "r");         //open
    if(!f) {
        printf("file %s not found:",location);
        exit;
    }


    int IN = fileno(f);


    int p[2]; 
    if(pipe(p) < 0)
    {
    close(p[0]);
    close(p[1]);
    close(IN);
    //perror("pipe:");
        exit;
    }
    
    if(fstat(IN, &stbuf) < 0)
    {
       // perror("fstat:");
        close(p[0]);
        close(p[1]);
        close(IN);
    exit;
    }
    lenData = stbuf.st_size;
    if ( AmountOfData != -1 && StartReadDataAt != -1 ){
                in_off=StartReadDataAt;
        if((StartReadDataAt+AmountOfData) > lenData){
            lenData= lenData-StartReadDataAt;
        }else{
                    lenData= AmountOfData;
        }
    }
    else if ( AmountOfData != -1 ){
        lenData= AmountOfData;
    }else if(StartReadDataAt != -1){
        in_off=StartReadDataAt;
        lenData=lenData-StartReadDataAt;
    }

       // fprintf( stderr, "lendData : %i\n", lenData);

    while(lenData>0)
    {

    if(buf_size > lenData){
        buf_size = lenData;
    }

    loff_t len = splice(IN, &in_off, p[1], NULL, buf_size, SPLICE_F_MOVE);  //input to p[1]
        if(len < 0)
        {
        close(p[0]);
            close(p[1]);
            close(IN);
            exit;
    }

        lenData=lenData-len;
    //fprintf( stderr, "lendData : %i\n", lenData);

        
        err = splice(p[0], NULL,STDOUT_FILENO, NULL, len, SPLICE_F_MOVE); //p[0] to STDOUT_FILENO

    }
    
    err = 0;

}

compile

gcc ./fastcat.c -o ./fastcat

example

./fastcat -f=100 -c=10000 /root/document