This topic was archived here:

https://web.archive.org/web/20230000000000fw_/https://www.zenhax.com/viewtopic.php?t=3063

eri619, posted Tue Sep 06, 2016 9:04 am (17260)

---

Well the heading says it all,i need quick bms to inject back to the decompressed bpe file,
here is the bpe source code,except in this case,it has been modified to work with Yukes games for Sony PS2..

Code:

#include 
#include 

#define BLOCKSIZE 10000    /* maximum block size */
#define HASHSIZE 8192       /* size of hash table */
#define MAXCHARS 220       /* char set per block */
#define THRESHOLD 3       /* minimum pair count */

unsigned char buffer[BLOCKSIZE];      /* data block */
unsigned char leftcode[256];      /* pair table */
unsigned char rightcode[256];      /* pair table */
unsigned char left[HASHSIZE];      /* hash table */
unsigned char right[HASHSIZE];      /* hash table */
unsigned char count[HASHSIZE];      /* pair count */
int size;      /* size of current data block */

/* function prototypes */
int lookup (unsigned char, unsigned char, int );
int fileread (FILE *, int, int, int);
void filewrite (FILE *);
void compress (FILE *, FILE *, int, int, int, int);

/* return index of character pair in hash table */
/* deleted nodes have a count of 1 for hashing */
int lookup (unsigned char a, unsigned char b, int hs)
{
  int index;   /* ?  - will add question marks until I understand each variable */


  /* compute hash key from both characters */
  index = (a ^ (b << 5)) & (hs-1);   /* ? */
/* if b = 10110101 then '(b << 5)' --> b = 10100000. */
/* ie shift the bits in b left by five positions and fill holes with zeros */

  /* search for pair or first empty slot */
  while ((left[index] != a || right[index] != b) && count[index] != 0)
  {
    index = (index   1) & (hs - 1);
  }

  left[index] = a;
  right[index] = b;
  return index;
}

/* read next block from input file into buffer */
int fileread (FILE *input, int bs, int hs, int mc)
{
  int c, index, used=0;

  /* reset hash table and pair table */
  for (c = 0; c < hs; c  )
    count[c] = 0;
  for (c = 0; c < 256; c  )
  {
    leftcode[c] = c;
    rightcode[c] = 0;
  }
  size = 0;

  /* read data until full or few unused chars */
  while (size < bs && used < mc && (c = getc(input)) != EOF)
  {
    if (size > 0)
    {
      index = lookup(buffer[size-1], c, hs);
      if (count[index] < 255) 
      {
          count[index];
      }
    }
    buffer[size  ] = c;

    /* use right code to flag data chars found */
    if (!rightcode[c])
    {
      rightcode[c] = 1;
      used  ;
    }
  }
  return c == EOF;
}

/* write each pair table and data block to output */
void filewrite( FILE *output )
{
  int i, len, c = 0;

  /* for each character 0..255 */
  while ( c < 256 ) 
  {
    /* if not a pair code, count run of literals */
    if ( c == leftcode[c] )
    {
      len = 1; c  ;
      while ( len < 127 && c < 256 && c == leftcode[c])
      {
        len  ; c  ;
      }
      putc( len   127, output ); 
      len = 0;
      if ( c == 256 ) break;
    }
   
    /* else count run of pair codes */
    else
    {
      len = 0;
      c  ;


/* original, will add extra brackets per compiler suggestions:      while ( len < 127 && c < 256 && c != leftcode[c] || len < 125 && c < 254 && c 1 != leftcode[c 1]) */
      while (( len < 127 && c < 256 && c != leftcode[c]) || (len < 125
&& c < 254 && c 1 != leftcode[c 1]))
      {
        len  ;
        c  ;
      }
      putc(len, output);
      c -= len 1;
    }

    /* write range of pairs to output */
    for ( i = 0; i <= len; i   )
    {
      putc(leftcode[c], output);
      if ( c != leftcode[c] )
      { putc(rightcode[c], output); }
      c  ;
    }
  }
  /* write size bytes and compressed data block */
  putc(size%6, output);
  putc(size/256, output);
  fwrite(buffer, size, 1, output);
}

/* compress from input file to output file */
void compress( FILE *infile, FILE *outfile, 
               int bs, int hs, int mc, int th )
{
  int leftch, rightch, code, oldsize;
  int index, r, w, best, done = 0;

  /* compress each data block until end of file */
  while ( !done )
  {
    done = fileread(infile, bs, hs, mc);
    code = 256;
   
    /* compress this block */
    for(;;)
    {
      /* get next unused chr for pair code */
      for ( code--; code >= 0; code-- )
      {
        if ( code == leftcode[code] && !rightcode[code] )
        {
          break;
        }
      }
 
      /* must quit if no unused chars left */
      if ( code < 0 )
      {
        break;
      }

      /* find most frequent pair of chars */
      for ( best = 2, index = 0; index < hs; index   )
      {
        if (count[index] > best) 
        {
          best = count[index];
          leftch = left[index];
          rightch = right[index];
        }
      }
    
      /* done if no more compression possible */
      if ( best < th )
      {
        break;
      }
   
      /* Replace pairs in data, adjust pair counts */
      oldsize = size - 1;
      for ( w = 0, r = 0; r < oldsize ; r   )
      {
        if (buffer[r] == leftch && buffer[r 1] == rightch)
        {
          if ( r > 0 )
          {
            index = lookup(buffer[w-1], leftch, hs);
            if ( count[index] > 1 )
            {
              --count[index];
            }
            index = lookup( buffer[w-1], code, hs );
            if ( count[index] < 255 ) 
            {
                count[index];
            }
          }
          if ( r < oldsize - 1 )
          {
            index = lookup( rightch, buffer[r 2] , hs);
            if ( count[index] > 1 )
            {
              --count[index];
            }
            index = lookup( code, buffer[r 2], hs );
            if ( count[index] < 255 ) 
            {
                count[index];
            }
          }
          buffer[w  ] = code;
          r  ;
          size--;
        }
        else
        {
          buffer[w  ] = buffer[r];
        }
      }
      buffer[w] = buffer[r];

      /* add to pair substitution table */
      leftcode[code] = leftch;
      rightcode[code] = rightch;
    
      /* delete pair from hash table */
      index = lookup( leftch, rightch, hs );
      count[index] = 1;
    }
    filewrite( outfile );
  }
}

void main (int argc, char *argv[] )
{
  FILE *infile, *outfile;
/* argc = 7              */
/*   argv[0] = command   */
/*   argv[1] = infile    */
/*   argv[2] = outfile   */
/*   argv[3] = BLOCKSIZE */
/*   argv[4] = HASHSIZE  */
/*   argv[5] = MAXCHARS  */
/*   argv[6] = THRESHOLD */
  int bs = 20000; /* maxval */
  int hs = 16384; /* maxval */
  int mc = 200;   /* default value */
  int th = 3;     /* default min */


  if (argc != 7)
  {
    printf("Usage: bpe infile outfile blocksize hashsize maxchars threshold\n");
    printf("typical: bpe infile outfile 5000 4096 200 3\n");
  }
  else
  {
    if (( infile = fopen( argv[1], "rb" )) == NULL )
    {
      printf("Error opening input %s\n",argv[1]);
    }
    else
    {
      if (( outfile = fopen( argv[2], "wb" )) == NULL )
      {
        printf("Error opening output %s\n",argv[2]);
      }
      else
      {
        bs = atoi( argv[3] );
        hs = atoi( argv[4] );
        mc = atoi( argv[5] );
        th = atoi( argv[6] );
        /* because these inputs come from the command line generated
           by clustor, I have included very little error checking here
        */    

        compress( infile, outfile, bs, hs, mc, th );
        fclose( outfile );
        fclose( infile );
      }
    }
  }
}

/* end of file */

aluigi, posted Tue Sep 06, 2016 10:32 am (17262)

---

That code you posted is the SAME of the one here:
https://github.com/wwylele/KEYTranslati ... /gfa/bpe.c

And I don't see references there about working with yuke/WWE, so, are you 100% sure?

aluigi, posted Tue Sep 06, 2016 10:34 am (17263)

---

And it's also the same identical implementation already available in quickbms!

eri619, posted Tue Sep 06, 2016 12:42 pm (17266)

---

aluigi wrote:

And it's also the same identical implementation already available in quickbms!

Yes i made modifications for it to work with WWE Yukes...

Modified two lines of code to make it work...could you add a inject feature for this code?

aluigi, posted Tue Sep 06, 2016 1:00 pm (17267)

---

What are these lines? The code is identical.

aluigi, posted Tue Sep 06, 2016 1:08 pm (17268)

---

Do you mean the endianess at line 139?

eri619, posted Tue Sep 06, 2016 1:15 pm (17269)

---

yes i modified the endianness at line 139, i switched places of these codes.
In the place of size/256 i replaced it with size%6 and in the place of size%6,i replaced it with size/256

aluigi, posted Tue Sep 06, 2016 1:21 pm (17270)

---

Ok, I can add a yuke_compress in the next quickbms.

eri619, posted Tue Sep 06, 2016 1:22 pm (17272)

---

aluigi wrote:

Ok, I can add a yuke_compress in the next quickbms.

Thank You,it would be really helpful.It can inject back with the same file size as original right?

aluigi, posted Tue Sep 06, 2016 2:38 pm (17275)

---

If you use the reimport feature you are bound to the size limits.
I'm not 100% sure if there are problems of compressed size like happens with lz4 and other algorithms, it depends by the decompression implementation.
What I mean is that the game may not accept the edited archive.

eri619, posted Tue Sep 06, 2016 5:35 pm (17279)

---

aluigi wrote:

If you use the reimport feature you are bound to the size limits.
I'm not 100% sure if there are problems of compressed size like happens with lz4 and other algorithms, it depends by the decompression implementation.
What I mean is that the game may not accept the edited archive.

Because thats what is wanted,SVR games accept compressed bpe files of any size,it has been tested.
But PS1 game Smackdown 2 requires the bpe file to be of its original size.The game reads only bpe file which is identical in size.

eatrawmeat391, posted Thu Sep 08, 2016 2:11 am (17342)

---

If you're going to add yuke compress support,remember to add the BPE Header,which is those 16 bytes below
The first 4 bytes: "BPE "
The next 4 bytes: "\x00\x01\x00\x00"
The next 4 bytes are compressed size in hex,the last 4 bytes are uncompressed size in hex.Size is in bytes

aluigi, posted Thu Sep 08, 2016 4:30 am (17345)

---

What you mention is part of the format and not part of the compression algorithm.

eri619, posted Sun Sep 11, 2016 7:21 am (17411)

---

aluigi wrote:

What you mention is part of the format and not part of the compression algorithm.

yes you are right

eri619, posted Sun Sep 11, 2016 7:22 am (17412)

---

eatrawmeat391 wrote:

If you're going to add yuke compress support,remember to add the BPE Header,which is those 16 bytes below
The first 4 bytes: "BPE "
The next 4 bytes: "\x00\x01\x00\x00"
The next 4 bytes are compressed size in hex,the last 4 bytes are uncompressed size in hex.Size is in bytes

There is no need to add a bpe header as quickbms injects the new file back into the old file provided the file sizes are the same

eri619, posted Wed Dec 07, 2016 9:58 am (19328)

---

has this been implemented in the new version yet?

aluigi, posted Wed Dec 07, 2016 2:54 pm (19333)

---

yuke_compress is available in quickbms 0.7.7, it uses the code of bpe_compress swapping the 16 bits values it writes.