Nodejs: convert string to buffer

13

I'm trying to write a string to a socket (socket is called "response"). Here is the code I have sofar (I'm trying to implement a byte caching proxy...):

var http = require('http');
var sys=require('sys');

var localHash={};

http.createServer(function(request, response) {
    var proxy = http.createClient(80, request.headers['host'])
    var proxy_request = proxy.request(request.method, request.url, request.headers);
    proxy_request.addListener('response', function (proxy_response) {
    proxy_response.addListener('data', function(x) {
        var responseData=x.toString();
        var f=50;
        var toTransmit="";
        var p=0;        

        var N=responseData.length;
        if(N>f){
            p=Math.floor(N/f);

            var hash="";
            var chunk="";
            for(var i=0;i<p;i++){
                chunk=responseData.substr(f*i,f);
                hash=DJBHash(chunk);
                if(localHash[hash]==undefined){
                    localHash[hash]=chunk;
                    toTransmit=toTransmit+chunk;
                }else{
                    sys.puts("***hit"+chunk);
                    toTransmit=toTransmit+chunk;//"***EOH"+hash;
                }
            }
            //remainder:
            chunk=responseData.substr(f*p);
            hash=DJBHash(chunk);
            if(localHash[hash]==undefined){
                localHash[hash]=chunk;
                toTransmit=toTransmit+chunk;
            }else{
                toTransmit=toTransmit+chunk;//"***EOH"+hash;
            }
        }else{
            toTransmit=responseData;
        }
        response.write(new Buffer(toTransmit));   /*error occurs here */
    });
    proxy_response.addListener('end', function() {
        response.end();
    });
    response.writeHead(proxy_response.statusCode, proxy_response.headers);
    });
    request.addListener('data', function(chunk) {
        sys.puts(chunk);
        proxy_request.write(chunk, 'binary');
    });
    request.addListener('end', function() {
        proxy_request.end();
    });
}).listen(8080);



function DJBHash(str) {
    var hash = 5381;
    for(var i = 0; i < str.length; i++) {
        hash = (((hash << 5) + hash) + str.charCodeAt(i)) & 0xffffffff;
    }
    if(hash<-1){
        hash=hash*-1;
    }
    return hash;
}

The trouble is, I keep getting a "content encoding error" in Firefox. It's as if the gizipped content isn't being transmitted properly. I've ensured that "toTransmit" is the same as "x" via console.log(x) and console.log(toTransmit).

It's worth noting that if I replace response.write(new Buffer(toTransmit)) with simply response.write(x), the proxy works as expected, but I need to do some payload analysis and then pass "toTransmit", not "x".

I've also tried to response.write(toTransmit) (i.e. without the conversion to buffer) and I keep getting the same content encoding error.

I'm really stuck. I thought I had this problem fixed by converting the string to a buffer as per another thread (http://stackoverflow.com/questions/7090510/nodejs-content-encoding-error), but I've re-opened a new thread to discuss this new problem I'm experiencing.

I should add that if I open a page via the proxy in Opera, I get gobblydeegook - it's as if the gzipped data gets corrupted.

Any insight greatly appreciated.

Many thanks in advance,

javascript
string
encoding
node.js
asked on Stack Overflow Aug 17, 2011 by Eamorr • edited Aug 17, 2011 by Eamorr

3 Answers

31

How about this?

var responseData = Buffer.from(x, 'utf8');

from: Convert string to buffer Node

answered on Stack Overflow Dec 10, 2016 by user1527225 • edited Feb 13, 2020 by Kirko
20

Without digging very deep into your code, it seems to me that you might want to change

var responseData=x.toString();

to

var responseData=x.toString("binary");

and finally

response.write(new Buffer(toTransmit, "binary"));
answered on Stack Overflow Aug 17, 2011 by Andris
5

From the docs:

Pure Javascript is Unicode friendly but not nice to binary data. When dealing with TCP streams or the file system, it's necessary to handle octet streams. Node has several strategies for manipulating, creating, and consuming octet streams.

Raw data is stored in instances of the Buffer class. A Buffer is similar to an array of integers but corresponds to a raw memory allocation outside the V8 heap. A Buffer cannot be resized.

So, don't use strings for handling binary data.

Change proxy_request.write(chunk, 'binary'); to proxy_request.write(chunk);.

Omit var responseData=x.toString();, that's a bad idea.

Instead of doing substr on a string, use slice on a buffer.

Instead of doing + with strings, use the "concat" method from the buffertools.

answered on Stack Overflow Aug 17, 2011 by thejh

User contributions licensed under CC BY-SA 3.0