HttpWebRequest, WebResponse stays empty

Hi,

I’m using a Fez Panda II version 4.1.5.0.

I am running the HttpWebRequest agains twitter for my ThermalTweeter project. The webrequest stays empty many times (but not always…). Anyone any idea what could be causing this?

When there is a result I get this:
{“completed_in”:0.022,“max_id”:98851831127613440,“max_id_str”:“98851831127613440”,“next_page”:“?page=2&max_id=98851831127613440&q=bieber&rpp=1”,“page”…

When there is no result, the debug just prints .

I already trimmed down ByteData and CharDate to prevent memory errors. I also query one tweet at a time to prevent overflowing the buffer. I’m also sure I’m not hammering search.twitter.com

Any idea?

Merijn


using System;
using System.IO;
using System.Threading;
using System.Text;
using Microsoft.SPOT;
using Microsoft.SPOT.Hardware;
using GHIElectronics.NETMF.FEZ;
using GHIElectronics.NETMF.Hardware;
using GHIElectronics.NETMF.Net;
using GHIElectronics.NETMF.Net.NetworkInformation;

namespace ThermalTweeter
{
    public class Program
    {
        static Thread thread;
        static bool running;
        //static UInt64 LastTwitterId, LastTwitterIdOld;

        public static void Main()
        {
            // Enable the Ethernet
            WIZnet_W5100.Enable(SPI.SPI_module.SPI1, (Cpu.Pin)FEZ_Pin.Digital.Di10, (Cpu.Pin)FEZ_Pin.Digital.Di7, true);
            Dhcp.EnableDhcp(new byte[] { 0x00, 0x26, 0x1C, 0x7B, 0x29, 0xE8 }, "tt");

            // Show we've started by blinking the LED
            int count = 0;
            bool ledState = false;
            OutputPort led = new OutputPort((Cpu.Pin)FEZ_Pin.Digital.LED, ledState);

            while (count < 20)
            {
                ledState = !ledState;
                led.Write(ledState);
                count++;
                Thread.Sleep(100);
            }

            // Begin reading twitter
            running = true;
            thread = new Thread(MainLoop);
            thread.Priority = ThreadPriority.Highest;
            thread.Start();
        }

        public static void MainLoop()
        {
            string result;

            while (running)
            {
                result = GetTweets("http://search.twitter.com/search.json?q=bieber&page=1&result_type=recent&rpp=1&show_user=true&since_id=");  
                Debug.Print(result); //prints null all the time!!!

                // first getting some results, then pagination, remembering last id, etc, etc

                Thread.Sleep(30000);
            }
        }

        public static string GetTweets(string url)
        {
            string result = null;

            // Create an HTTP Web request.
            HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;

            // Set request.KeepAlive to use a persistent connection. 
            request.KeepAlive = false;

            // Get a response from the server.
            WebResponse resp = null;
            try
            {
                resp = request.GetResponse();
            }
            catch (Exception e)
            {
                Debug.Print("Exception in HttpWebRequest.GetResponse(): " + e.ToString());
            }

            // Get the network response stream to read the page data.
            if (resp != null)
            {
                Stream respStream = resp.GetResponseStream();
                Debug.GC(true);
                Thread.Sleep(1000);
                byte[] byteData = new byte[1596]; //trimmed down for FEZ Panda II
                char[] charData = new char[1596]; 
                int bytesRead = 0;
                Decoder UTF8decoder = System.Text.Encoding.UTF8.GetDecoder();
                int totalBytes = 0;

                // allow 5 seconds for reading the stream
                respStream.ReadTimeout = 5000;

                // If we know the content length, read exactly that amount of 
                // data; otherwise, read until there is nothing left to read.
                if (resp.ContentLength != -1)
                {
                    for (int dataRem = (int)resp.ContentLength; dataRem > 0; )
                    {
                        Thread.Sleep(500);
                        bytesRead = respStream.Read(byteData, 0, byteData.Length);
                        if (bytesRead == 0)
                        {
                            Debug.Print("Error: Received " + (resp.ContentLength - dataRem) + " Out of " + resp.ContentLength);
                            break;
                        }
                        dataRem -= bytesRead;

                        // Convert from bytes to chars, and add to the page string.
                        int byteUsed, charUsed;
                        bool completed = false;
                        totalBytes += bytesRead;
                        UTF8decoder.Convert(byteData, 0, bytesRead, charData, 0, bytesRead, true, out byteUsed, out charUsed, out completed);
                        result = result + new String(charData, 0, charUsed);
                    }

                    result = new String(System.Text.Encoding.UTF8.GetChars(byteData));
                }
                else
                {
                    // Read until the end of the data is reached.
                    while (true)
                    {
                        // If the Read method times out, it throws an exception, 
                        // which is expected for Keep-Alive streams because the 
                        // connection isn't terminated.
                        try
                        {
                            Thread.Sleep(500);
                            bytesRead =
                                respStream.Read(byteData, 0, byteData.Length);
                        }
                        catch (Exception)
                        {
                            bytesRead = 0;
                        }

                        // Zero bytes indicates the connection has been closed by the server.
                        if (bytesRead == 0)
                            break;

                        int byteUsed, charUsed;
                        bool completed = false;
                        totalBytes += bytesRead;
                        UTF8decoder.Convert(byteData, 0, bytesRead, charData, 0, bytesRead, true, out byteUsed, out charUsed, out completed);
                        result = result + new String(charData, 0, charUsed);
                    }
                }

                // Close the response stream.  For Keep-Alive streams, the 
                // stream will remain open and will be pushed into the unused 
                // stream list.
                resp.Close();
            }
            return result;
        }
    }
}

At the end of your Main() method add:

Thread.Sleep(Timeout.Infinite);

You are exiting the main thread. Funny things can happen.

@ Mike: Thanks for the tip!

Tried it, however it dit not solve it, too bad. The webresponse keeps coming back with no bytes. However sometimes a result occurs. Hope someone has an idea, because I would really like my Fez Panda to consume HTTP well.

Maybe use a network analyzer to see what is happening?

Have you tried putting in a break point and looking at the headers? Maybe you’re getting a 301 or similar

there are plenty of projects showing that the Panda and Wiz 5100 combination can be made to do lots of cool things. So if anything, I’d say look through some of those (like the network configuration project [url]http://code.tinyclr.com/project/308/netconfig-wiz5100-http-configuration-framework/[/url]) and see what they do and if there’s anything you’re doing differently…

Has anybody gotten HttpWebRequest to work properly when using PUT or POST? I have seen examples that use GET, but nothing with PUT or POST. I have written code that works for ThingSpeak and SmartEnergyGroups and if I use a socket directly can get a proper response, but using HttpWebRequest and WebResponse, I get 0 for the response code. Using WireShark I can see that (at least ThingSpeak) does correctly respond with a 200 response code in both cases (socket or web request), so it seems that WebRequest WebResponse is not working for me with a Panda II and Connect (WIZnet). In all cases (socket or web request, ThingSpeak or SmartEnergyGroups) the POST/PUT gets to where it needs to, but the code can’t easily tell that it was successful. Yes, I could process the data from the socket directly to figure it out, but it is easier to just look at the response code and possibly also parse the response as necessary.

After much time (but only a little debugging) I think I have a better idea of whats going on and it looks like it might be a previously undocumented feature of the WIZnet 5100 implementation. I used Wireshark to look at the network traffic and it was not quite what I expected. There was an http [RST] shortly after the PUT was sent on the wire. This did not show up (with the same timing) when using the socket implementation.

I then converted the code so it would run on the emulator and (of course) it worked fine.

Since it looked like it could be a timing issue, I decided to try the old embedded trick of waiting a while. When I put a sleep of about a second between writing the stream (that I got from GetRequestStream) and GetResponse everything started working.

I discovered HaveResponse() and tried it, but each time I checked it was false, but it still worked because this took long enough with the sleeps I put in it.

I then tried checking HaveResponse() after GetResponse(), but each of the ten times I checked showed false. However, when I used the response it was fine, since about a second had passed between writing the stream and using the response.

It certainly appears that the implementation of HttpWebRequest is not quite the same as it is in normal .Net. Any ideas?

Has anyone got it to work? All the examples that I saw were not sending POSTs or PUTs with data and then getting a response.

I remember this example uses PUT or POST. I am not sure:
[url]http://wiki.tinyclr.com/index.php?title=WIZnet_W5100_HTTP_FileServer[/url]

Hi Joe,

That was a good tip, but it uses the server version of PUT and POST. The issue I am having is with the client version of PUT and POST. I am using if for Smart Energy Groups and also ThingSpeak. If I don’t wait about a second before trying to request.GetResponse() after writing the stream, the stream that I get for reading has not been parsed correctly and includes the headers. If I wait the second, it appears to be correct. The Thread.Sleep(1000) is not needed in the emulator version. Also, HaveResponse() always returns False, even when there is a response that is correct.

I am now fairly convinced there is an issue with the WIZnet implementation of WebRequest.GetResponse(). It seems that if I do not allow enough time to pass before calling GetResponse(), I either get no response or I get an unprocessed response.

I found a test post site that makes it easy to verify if things are working correctly or not. If I comment out the Thread.Sleep(10), there is no repsonse. If I leave it there, the repsonse that is printed is the whole repsonse from the server with all of the headers and the StatusCode and ContentLength are 0. If I increase the delay to 1000, the repsonse no longer has the headers in it and StatusCode and ContentLength are correct.

While I have a workaround, I am not convinced that 1000 will always be enough time and it does seem like there is something that is not working as it should.

I also noticed that HaveResponse always returns False, but it does this in the emulator too, so this might be a netmf bug.

Perhaps somebody could try this on an EMX, to see if it has the same problem?


using System;
using System.IO;
using System.Net;
using System.Text;
using System.Threading;
using Microsoft.SPOT;
using Microsoft.SPOT.Hardware;
using GHIElectronics.NETMF.FEZ;
using GHIElectronics.NETMF.Hardware;
using GHIElectronics.NETMF.Net;
using GHIElectronics.NETMF.Net.NetworkInformation;

namespace MFConsoleApplication1
{
    public class Program
    {
        public static void Main()
        {
            WIZnet_W5100.Enable(SPI.SPI_module.SPI1, (Cpu.Pin)FEZ_Pin.Digital.Di10, (Cpu.Pin)FEZ_Pin.Digital.Di7, true);
            Dhcp.EnableDhcp(new byte[] { 0x00, 0x90, 0x04, 0x01, 0x20, 0x03 }, "Fez");

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://posttestserver.com/post.php?dir=tiny") as HttpWebRequest;
            request.Method = "PUT";
            request.KeepAlive = false;
            request.Headers.Add("Content-Type: application/x-www-form-urlencoded");
            request.ReadWriteTimeout = 10000; // 10 second timeout

            string postData = "data_post=(site abcd (node zyx ? (a 1) (b 34)))";

            String page = null;
            Byte[] bytesToSend = Encoding.UTF8.GetBytes(postData);
            request.ContentLength = bytesToSend.Length;
 
            using (Stream newStream = request.GetRequestStream())
            {
                newStream.Write(bytesToSend, 0, bytesToSend.Length);
            }
            Thread.Sleep(10);
            using (HttpWebResponse resp = (HttpWebResponse)request.GetResponse())
            {
                if (null != resp)
                {
                    using (Stream respStream = resp.GetResponseStream())
                    {
                        using (StreamReader readStream = new StreamReader(respStream))
                        {
                            char[] charData = new char[500];
                            int count = readStream.Read(charData, 0, 500);
                            page = new String(charData, 0, count);
                            Debug.Print("Response ContentLength: " + resp.ContentLength);
                            Debug.Print("Response StatusCode: " + resp.StatusCode);
                            Debug.Print("Returned =" + page);
                        }
                    }
                }
            }
            Thread.Sleep(100000);


        }

    }
}

I got to try the example above on a CANxtra (with a few changes for the built-in ethernet) and it works fine both with and without the Thread.Sleep() before the GetResponse(). At this point I am pretty convinced there is a bug in the WIZnet W5100 implementation of HttpWebRequest GetResponse(). There does seem to be a reasonable workaround, but I would like to know if this will be fixed in a future version of the SDK?

Thank you for the feedback Frogmore.