Transfer Large binary data from client to server uses too much heap memory

I wrote a java client to send a 95mb of file to a hessian servlet using
the latest version of hessian.

The problem i have is that the client needs at least 700mb of heap in
order to send. Otherwise, i receive OutOfMemoryError.

Please see the attached sources. Any hints are mostly welcome.

Thanks a lot.

package interop;

import com.caucho.hessian.io.HessianInput;
import com.caucho.hessian.io.HessianOutput;

import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.ServletOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.FileOutputStream;
import java.io.File;
import java.nio.channels.WritableByteChannel;
import java.nio.ByteBuffer;
import java.lang.reflect.Method;

public class FileServlet extends HttpServlet
{
    protected void doPost(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) throws ServletException, IOException
    {
        System.out.println("FileServlet.doPost");

        final ServletInputStream inputStream = httpServletRequest.getInputStream();
        System.out.println("a");
        final ServletOutputStream outputStream = httpServletResponse.getOutputStream();
        System.out.println("b");
        HessianInput in = new HessianInput(inputStream);
        System.out.println("c");
        HessianOutput out = new HessianOutput(outputStream);
        System.out.println("d");

        final int i = in.readCall();
        System.out.println("i = " + i);

        String header;
            while ((header = in.readHeader()) != null) {
              Object value = in.readObject();
              System.out.println("value = " + value);
            }

            String methodName = in.readMethod();
        System.out.println("methodName = " + methodName);

        final File fileName = new File(in.readString());
        System.out.println("fileName = " + fileName);
        final InputStream fileIs = in.readInputStream();
        System.out.println("fileIs = " + fileIs);

        boolean success = false;
        try
        {
            writeToFile(fileName, fileIs);
            success =true;
        }
        catch (Exception e)
        {
            success = false;
        }
        finally
        {
            System.out.println("success = " + success);
            in.completeCall();
            out.startReply();
            out.writeBoolean(success);
            out.completeReply();
        }
    }

    private void writeToFile(File fileName, InputStream fileIs)
    {
        System.out.println("fileName = " + fileName);
        try {
                // Obtain a channel
                WritableByteChannel channel = new FileOutputStream(fileName).getChannel();

                // Create a direct ByteBuffer;
                // see also e158 Creating a ByteBuffer
            int size=1024*1024;
            ByteBuffer buf = ByteBuffer.allocateDirect(size);

                byte[] bytes = new byte[size];
                int count = 0;
                int index = 0;

                // Continue writing bytes until there are no more
                while (count >= 0) {
                    if (index == count) {
                        count = fileIs.read(bytes);
                        index = 0;
                    }
                    // Fill ByteBuffer
                    while (index < count && buf.hasRemaining()) {
                        buf.put(bytes[index++]);
                    }

                    // Set the limit to the current position and the position to 0
                    // making the new bytes visible for write()
                    buf.flip();

                    // Write the bytes to the channel
                    int numWritten = channel.write(buf);
                    // Check if all bytes were written
                    if (buf.hasRemaining()) {
                        // If not all bytes were written, move the unwritten bytes
                        // to the beginning and set position just after the last
                        // unwritten byte; also set limit to the capacity
                        buf.compact();
                    } else {
                        // Set the position to 0 and the limit to capacity
                        buf.clear();
                    }
                }

                // Close the file
                channel.close();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
    }
}

package interop;

import com.caucho.hessian.io.HessianInput;
import com.caucho.hessian.io.HessianOutput;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;

public class FileClient
{
    public static void main(String[] args)
            throws Throwable
    {
        URL url = new URL("http://localhost:8080/ribena/FileServlet");
        HttpURLConnection conn;
        conn = (HttpURLConnection) url.openConnection();

        conn.setRequestMethod("POST");
        OutputStream os;

        conn.setDoOutput(true);
        os = conn.getOutputStream();

        HessianOutput out = new HessianOutput(os);
        out.startCall("upload");
        final String shortFileName;
       // shortFileName = "airwave.txt";
       shortFileName = "VP_Suite_Windows_1_1_20050222.exe";

        out.writeString(shortFileName);
        File file = new File(shortFileName);
        uploadNIO(file, out);
        out.completeCall();


        InputStream is = conn.getInputStream();

        HessianInput in = new HessianInput(is);

        final boolean ok = ((Boolean) in.readReply(Boolean.class)).booleanValue();
        System.out.println("ok = " + ok);
        in.completeValueReply();
    }

    private static void uploadNIO(File file, HessianOutput out) throws IOException
    {

        out.writeByteBufferStart();
                System.out.println("file = " + file.exists());
        int count=0;
        try
        {
            // Obtain a channel
            ReadableByteChannel channel = new FileInputStream(file).getChannel();

            // Create a direct ByteBuffer; see also e158 Creating a ByteBuffer
            final int size = 1024 * 1024*10;
            ByteBuffer buf = ByteBuffer.allocateDirect(size);
            byte [] tmpByteArray = new byte[size];

            int numRead = 0;
            while (numRead >= 0)
            {
                count++;
                System.out.println("count = " + count);
                // read() places read bytes at the buffer's position so the
                // position should always be properly set before calling read()
                // This method sets the position to 0
                buf.rewind();

                System.out.println("buf.rewind()");

                // Read bytes from the channel
                numRead = channel.read(buf);

                System.out.println("numRead = channel.read(buf)");

                // The read() method also moves the position so in order to
                // read the new bytes, the buffer's position must be set back to 0
                buf.rewind();

                System.out.println("buf.rewind()");

                // Read bytes from ByteBuffer; see also
                // e159 Getting Bytes from a ByteBuffer
                if (numRead > 0)
                {
                    //final byte[] tmpByteArray = buf.array();
                    //byte [] tmpByteArray = new byte[size];
                    buf.get(tmpByteArray);
                    System.out.println("got");
                    out.writeByteBufferPart(tmpByteArray,0,numRead);
                }

                System.out.println("writeByteBufferPart");

            }
            out.writeByteBufferEnd(new byte[0],0,0);
            System.out.println("count*size = " + count * size);
        }
        catch (Error e)
        {
            e.printStackTrace();
            throw new RuntimeException(e);
        }
    }
}
Received on Sun 24 Apr 2005 18:26:34 -0700