Main Site Documentation

Panda II: remote XML parsing problem


#1

Hi all,

I have a problem with parsing XML document downloaded via webrequest. Here is my code:


            HttpWebRequest WebReq = null;
            WebResponse resp = null;
            Stream respStream = null;
            StreamReader readStream = null;
            String all = null;
            XmlReader xml = null;

            while (true)
            {
                // Sleep for 500 milliseconds
                Thread.Sleep(2000);

                try
                {
                    WebReq = (HttpWebRequest)WebRequest.Create(url);
                    resp = WebReq.GetResponse();

                    respStream = resp.GetResponseStream();
                    readStream = new StreamReader(respStream);
                    
                    xml = XmlReader.Create(respStream);
                    System.Text.Encoding enc = readStream.CurrentEncoding; // really UTF8

                    while (xml.Read())
                    {
                        if (xml.NodeType == XmlNodeType.Element && xml.Name == "time") {
                        }
                    }
                }
                catch (Exception e)
                {
                }
                xml.Close();
                resp.Close();
                respStream.Close();
                readStream.Close();
                WebReq.Dispose();
                all = null;
            }
        }

xml.Read() throw the exception UnknownEncoding, however, downloaded document is utf8 encoded. I put there readStream variable from which I get CurrentEncoding and it is UTF8. When I put the “url” directly to webbrowser than proper XML document is shown with UTF8 encoding. Have you any idea where I am wrong? Thank you very much…


#2

Have you tried to follow this code in the debugger to see if it is failing on first read or after multiple reads?


#3

In my project I use next code for xml data sending&receiving:


string str = new string("<?xml version=\"1.0\" encoding=\"UTF-8\"?><packet><to>Tove</to><from>Jani</from><heading>Reminder</heading><body>Don't forget me this weekend!</body></packet>");
			string URI = "http://www.yourdomain.com";
            byte[] buffer = Encoding.UTF8.GetBytes(str);
            HttpWebRequest WebReq = (HttpWebRequest)WebRequest.Create(URI);
            WebReq.Method = "POST";
            WebReq.ContentType = "text/xml";
            WebReq.ContentLength = buffer.Length;
            Stream PostData = WebReq.GetRequestStream();
            PostData.Write(buffer, 0, buffer.Length);
            PostData.Close();
            HttpWebResponse WebResp = (HttpWebResponse)WebReq.GetResponse();
            Stream Answer = WebResp.GetResponseStream();
			XmlReaderSettings ss = new XmlReaderSettings();
            ss.IgnoreWhitespace = true;
            ss.IgnoreComments = false;
            XmlReader xmlr = XmlReader.Create(Answer, ss);
            while (xmlr.Read())
            {
                if (xmlr.NodeType == XmlNodeType.Element && xmlr.LocalName == "packet")
                {
					xmlr.MoveToElement();
                    xmlr.ReadToDescendant("to");
                    string to = xmlr.ReadElementString();
                    Debug.Print("to:" + to);
                    xmlr.MoveToContent();
                    string from = xmlr.ReadElementString();
                    Debug.Print("from:" + from);
                    xmlr.MoveToContent();
                    string heading = xmlr.ReadElementString();
                    Debug.Print("heading:" + heading);
                    xmlr.MoveToContent();
                    string body = xmlr.ReadElementString();
                    Debug.Print("body:" + body);
                }
            }
            xmlr.Close();
            Answer.Close();

So try remove:
readStream = new StreamReader(respStream);
add:
XmlReaderSettings


#4

To Mike:

Yes, I tried it and it fails on first read.

To Dejan:
Thanks, I will try your code and will see if it helps…


#5

Let us know if you get it working.


#6

Hi,

this is update about my issue. I tried to modified my sourcecode according to your recomandation. This is the code:


        public static bool SendToServer(string url, string data)
        {
            bool result = false;

            HttpWebRequest WebReq = null;
            WebResponse WebResp = null;
            Stream respStream = null;
            XmlReader xml = null;

            try
            {
                byte[] buffer = Encoding.UTF8.GetBytes("data=" + data);
                WebReq = (HttpWebRequest)WebRequest.Create(url);
                WebReq.Method = "POST";
                WebReq.ContentType = "application/x-www-form-urlencoded";
                WebReq.ContentLength = buffer.Length;
                Stream PostData = WebReq.GetRequestStream();     // start problem
                PostData.Write(buffer, 0, buffer.Length);
                PostData.Close();
                WebResp = WebReq.GetResponse();
                respStream = WebResp.GetResponseStream();         // end problem
                XmlReaderSettings ss = new XmlReaderSettings();
                ss.IgnoreWhitespace = true;
                ss.IgnoreComments = false;
                xml = XmlReader.Create(respStream, ss);
                while (xml.Read())
                {
                    if (xml.NodeType == XmlNodeType.Element && xml.Name == "time")
                    {
                        // some not important code
                    }
                }
                result = true;
            }
            catch (Exception e)
            {
                result = false;
            }

            if (xml != null)
                xml.Close();
            if (respStream != null)
                respStream.Close();

            return result;
        }

The problem was a little bit more complicated. There was also a problem on server (PHP) during the XML generating (no problems in webbrowser). I don’t understand why, but when I commented out one function the problem with UnknownEncoding disapeared. That is optimistic :slight_smile:

However, another problem occured. Very rarely the code throw an exception UnexpectedEOF (in first call of xml.Read() function). But more serious problem is with exception Socket is closed somewhere between comments “start problem” and “end problem”. This exception occures mostly on line with WebRequestStream however sometime later in the code. Never after function GetResponseStream…

Do you have any idea what the problem is? I am still trying. I Hope I will finaly make it works with your help…

PS: The parametr data in the function above is regular xml document in string.


#7

Few questions.
Do you use own remote server and own php scripts? If yes then I in your case will use another approach.
In client side I will send with POST request xml document as RequestStream. In server side I will use php libxml extension:
http://php.net/manual/en/book.libxml.php
On server side I will use script like this:


<?php
	include('config.php');
	
	libxml_use_internal_errors(true);
	libxml_clear_errors();

	$db = mysql_connect($dbhost, $dbuser, $dbpasswd);
	mysql_select_db ($dbname) or die ("Cannot connect to database");
	mysql_query("SET NAMES 'utf8'");
	
	try {
		$request_xml = new SimpleXMLElement($HTTP_RAW_POST_DATA);
	} catch (Exception $e) {}
	
	if (!$request_xml)
	{
		$response_xml = new SimpleXMLElement("<?xml version='1.0' encoding=\"utf-8\" ?><error></error>");
		foreach(libxml_get_errors() as $error)
		{
			$response_xml->addChild('message', trim($error->message));
		}
		libxml_clear_errors();
		header('Content-Type: text/xml');
		echo $response_xml->asXML();
	}
	else
	{
		$to = mysql_real_escape_string($request_xml->to);
		$from = mysql_real_escape_string($request_xml->from);
		$heading = mysql_real_escape_string($request_xml->heading);
		$body = mysql_real_escape_string($request_xml->body);
		
		$sql = "INSERT INTO table ...";
		$result = mysql_query($sql);
		$device = mysql_fetch_object($result);

		$response_xml = new SimpleXMLElement("<?xml version='1.0' encoding=\"utf-8\" ?><packet></packet>");
		$response_xml->addChild('to', $to);
		$response_xml->addChild('from', $from);
		$response_xml->addChild('heading', $heading);
		$response_xml->addChild('body', $body);

		header('Content-Type: text/xml');
		echo $response_xml->asXML();
	}
	mysql_free_result($result);
	mysql_close($db);
?>

Try replace:

WebResponse WebResp = null;

with:

HttpWebResponse WebResp = null;

#8

Hi,

bad news… I created the most simple XML:


<?php 
libxml_use_internal_errors(true);
libxml_clear_errors();

try {
  $request_xml = new SimpleXMLElement($HTTP_RAW_POST_DATA);
} catch (Exception $e) {}
 
	
$response_xml = new SimpleXMLElement("<?xml version='1.0' encoding=\"utf-8\" ?><data></data>");
$response_xml->addChild('time', date("Y-m-d-H-i-s"));

header('Content-Type: text/xml');
echo $response_xml->asXML();
	
?>

The micro framework code is the same with little modification. I change WebResponse to HttpWebResponse but with no success. At the end I added WebResp.Close(). If I am posting data with stream postData than every second response throws an exception “Socket is closed”. If I remove these data and make request with no posted data than almost every second respons throws an exception UnexpectedEOF. Hm… I don’t know what else I can do.


#9

Hi,
Im make you demo which work ok for me(On EMX). I don’t know if in PANDA II mising some functionality and XML’s doesn’t work…
Instructions:
1.)Download example from: Mediafire link and extract it.
2.)Copy test.php to your server and test if file work ok. You can use Firefox REST client:
Method: POST
URL: http://www.yourdomain.com/test.php
Request body:


<?xml version="1.0" encoding="utf-8"?><packet><ping>This is test message</ping></packet>

In your response body you must see:

<?xml version="1.0" encoding="utf-8"?><packet><response>This is test message</response></packet>

3.)Create new project.
4.)Copy and add “ServerComunicaton.cs” to your project.
5.) Your main class must look like this:


using System.IO;
using System.Net;
using System.Xml;
using System.Ext.Xml;
using Microsoft.SPOT;
using XMLTest;

namespace Test
{
    public class Program
    {
        public static void Main()
        {
            ServerComunicaton sc = new ServerComunicaton();
            XmlReader xmlr = sc.RequestTest();
            while (xmlr.Read())
            {
                if (xmlr.NodeType == XmlNodeType.Element && xmlr.LocalName == "packet")
                {
                    xmlr.MoveToElement();
                    xmlr.ReadToDescendant("response");
                    string response = xmlr.ReadElementString();
                    Debug.Print("response:" + response);
                }
            }
            xmlr.Close();
            while (true)
            {

            }
        }        
    }
}

6.)Build&Deploy and you will get in VS Debug window:


Request: <?xml version="1.0" encoding="utf-8"?>
<packet><ping>This is test message</ping></packet>
response:This is test message

If you get exception then you have some problems with server or network connection on Panda. Try ping server from panda.


#10

Hi Dejan,

you are very kind that you are helping me. However, I have bad news. I tried your code and made some stress tests - in cycle with 5000ms. It is strange. Sometime it works 50 times in a row. Than it starts to throw exceptions. And than it works OK again. I don’t understand. I tried the example against localhost and also against regular server on remote webhosting.

It trows xmlException UnexpectedEOF. Only sometimes it throws Socket is closed exception. :frowning:

PS: I added to your code only initialization of connection.


public ServerComunicaton()
        {
            byte[] ip = { 192, 168, 1, 176 };
            byte[] subnet = { 255, 255, 255, 0 };
            byte[] gateway = { 192, 168, 1, 1 };
            byte[] mac = { 0x00, 0x88, 0x98, 0x90, 0xD4, 0xE0 };
            WIZnet_W5100.Enable(SPI.SPI_module.SPI1, (Cpu.Pin)FEZ_Pin.Digital.Di10, (Cpu.Pin)FEZ_Pin.Digital.Di7, true);

            NetworkInterface.EnableStaticIP(ip, subnet, gateway, mac);
            NetworkInterface.EnableStaticDns(gateway);
        }


#11

I believe you are running into the same problem with the WIZnet library that I did. What I found to work was adding:


            Debug.Print("waiting before GetResponse()");
            Thread.Sleep(1000);

After writing the request stream and before getting the response. I experimented with the value and this seemed to work all of the time. Smaller values worked sometimes, but the results were inconsistent. Sometimes the response would be correct and other times it would have the HTTP headers in it.


#12

Hi Frogmore,

I am affraid it didn’t help. Still the same problem - Socket is closed while sending the request or UnexpectedEOF while reading the XML. :frowning:


#13

Hi all,

I have good and bad news :slight_smile: I finaly solved the problem. I got 100% successful request. Never done before! :slight_smile: The bad thing is I don’t know how :slight_smile: This is the code:


using System;
using System.IO;
using System.Threading;
using System.Text;
using System.Xml;
using System.Ext.Xml;
using Microsoft.SPOT;
using Microsoft.SPOT.Hardware;
using GHIElectronics.NETMF.FEZ;
using GHIElectronics.NETMF.Hardware;
using GHIElectronics.NETMF.Net;
using GHIElectronics.NETMF.Net.NetworkInformation;

namespace test3
{
    public class Program
    {
        static Thread thread;
        static bool running;

        public static void Main()
        {
            // Enable the Ethernet
            WIZnet_W5100.Enable(SPI.SPI_module.SPI1, (Cpu.Pin)FEZ_Pin.Digital.Di10, (Cpu.Pin)FEZ_Pin.Digital.Di7, true);
            Dhcp.EnableDhcp(new byte[] { 0x00, 0x26, 0x1C, 0x7B, 0x29, 0xE8 }, "tt");

            // Begin reading twitter
            running = true;
            thread = new Thread(MainLoop);
            thread.Priority = ThreadPriority.Highest;
            thread.Start();
        }

        public static void MainLoop()
        {
            string result;

            while (running)
            {
                result = SendToServer("http://192.168.1.100/test.php");
                Debug.Print(result);
                Thread.Sleep(5000);
            }
        }

        public static string SendToServer(string url)
        {
            string result = null;

            MemoryStream ms = new MemoryStream();
            XmlWriter xmlwrite = XmlWriter.Create(ms);

            xmlwrite.WriteProcessingInstruction("xml", "version=\"1.0\" encoding=\"utf-8\"");
            xmlwrite.WriteStartElement("packet");//root element
            xmlwrite.WriteStartElement("ping");
            xmlwrite.WriteString("This is test message");
            xmlwrite.WriteEndElement();
            xmlwrite.WriteEndElement();//end the root element
            xmlwrite.Flush();
            xmlwrite.Close();
            //////// display request XML data ///////////
            byte[] byteArray = ms.ToArray();
            char[] cc = System.Text.UTF8Encoding.UTF8.GetChars(byteArray);
            string str = new string(cc);
            Debug.Print("Request: " + str);

            byte[] buffer = Encoding.UTF8.GetBytes(str);

            HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
            request.KeepAlive = false;

            request.Method = "POST";
            request.ContentType = "text/xml";
            request.ContentLength = buffer.Length;
            Stream PostData = request.GetRequestStream();
            PostData.Write(buffer, 0, buffer.Length);
            PostData.Close();

            WebResponse resp = null;
            try
            {
                resp = request.GetResponse();
            }
            catch (Exception e)
            {
                Debug.Print("Exception in HttpWebRequest.GetResponse(): " + e.ToString());
            }

            Stream Answer = resp.GetResponseStream();
            XmlReaderSettings ss = new XmlReaderSettings();
            ss.IgnoreWhitespace = true;
            ss.IgnoreComments = false;
            XmlReader xmlr = XmlReader.Create(Answer, ss);
            Answer.Close();

            try
            {
                while (xmlr.Read())
                {
                    if (xmlr.NodeType == XmlNodeType.Element && xmlr.LocalName == "packet")
                    {
                        xmlr.MoveToElement();
                        xmlr.ReadToDescendant("response");
                        string response = xmlr.ReadElementString();
                        Debug.Print("response:" + response);
                    }
                    Debug.Print("XML OK");
                }
            }
            catch (System.Exception e)
            {
                Debug.Print("Exception: " + e.Message);
            }
            xmlr.Close();

            return result;
        }
    }
}

I used DHCP, threads and some other minor changes. But the core is the same. I don’t know where is the key difference.


#14

It seems, that the problem with exception Socket is closed was eliminated by:

request.KeepAlive = false;

Now I have to figure out what was causing the UnexpectedEOF exception :slight_smile:

Good night and thank you…


#15

Are you shure that when you are receive UnexpectedEOF that XML structure is valid?
You can Debug.Prin your response from server and see if is realy only XML or it include also some other things. You can do this if you replace in ServerCommunication.cs lines:


            Stream Answer = WebResp.GetResponseStream();
            XmlReaderSettings ss = new XmlReaderSettings(); 

with:


            Stream Answer = WebResp.GetResponseStream();
            StreamReader reader = new StreamReader(Answer);
            Debug.Print("Response Stream: " + reader.ReadToEnd());
            reader.Close();
            XmlReaderSettings ss = new XmlReaderSettings(); 

You can also try add validate option to XmlReaderSettings …