XmlReader BINARY_BLOB_HEAD memory problem

NETMF & Gadgeteer
1

I made a program thats read xml files from a usb stick and collect specific data from it.
But after collecting the data the BINARY_BLOB_HEAD keep getting bigger and after each run until I run out of memory.
Does anyone know how I can solve this problem?

Started:

GC: 3msec 34536 bytes used, 29844 bytes available
Type 0F (STRING ): 1152 bytes
Type 11 (CLASS ): 4008 bytes
Type 12 (VALUETYPE ): 72 bytes
Type 13 (SZARRAY ): 2472 bytes
Type 15 (FREEBLOCK ): 29844 bytes
Type 16 (CACHEDBLOCK ): 324 bytes
Type 17 (ASSEMBLY ): 13404 bytes
Type 18 (WEAKCLASS ): 48 bytes
Type 19 (REFLECTION ): 24 bytes
Type 1B (DELEGATE_HEAD ): 468 bytes
Type 1C (DELEGATELIST_HEAD ): 48 bytes
Type 1D (OBJECT_TO_EVENT ): 96 bytes
Type 1E (BINARY_BLOB_HEAD ): 8304 bytes
Type 1F (THREAD ): 1536 bytes
Type 20 (SUBTHREAD ): 192 bytes
Type 21 (STACK_FRAME ): 1020 bytes
Type 27 (FINALIZER_HEAD ): 96 bytes
Type 31 (IO_PORT ): 72 bytes
Type 34 (APPDOMAIN_HEAD ): 72 bytes
Type 36 (APPDOMAIN_ASSEMBLY ): 1128 bytes

Ended:

C: 3msec 53976 bytes used, 10404 bytes available
Type 0F (STRING ): 2088 bytes
Type 11 (CLASS ): 5616 bytes
Type 12 (VALUETYPE ): 72 bytes
Type 13 (SZARRAY ): 2472 bytes
Type 15 (FREEBLOCK ): 10404 bytes
Type 17 (ASSEMBLY ): 13404 bytes
Type 18 (WEAKCLASS ): 48 bytes
Type 19 (REFLECTION ): 24 bytes
Type 1B (DELEGATE_HEAD ): 468 bytes
Type 1C (DELEGATELIST_HEAD ): 48 bytes
Type 1D (OBJECT_TO_EVENT ): 96 bytes
Type 1E (BINARY_BLOB_HEAD ): 25296 bytes
Type 1F (THREAD ): 1536 bytes
Type 20 (SUBTHREAD ): 192 bytes
Type 21 (STACK_FRAME ): 1248 bytes
Type 27 (FINALIZER_HEAD ): 96 bytes
Type 31 (IO_PORT ): 72 bytes
Type 34 (APPDOMAIN_HEAD ): 72 bytes
Type 36 (APPDOMAIN_ASSEMBLY ): 1128 bytes
Failed allocation for 89 blocks, 1068 bytes

#### Exception System.OutOfMemoryException - CLR_E_OUT_OF_MEMORY (3) ####
#### Message: 
#### Microsoft.SPOT.IO.NativeFileStream::.ctor [IP: 0000] ####
#### System.IO.FileStream::.ctor [IP: 00fd] ####
#### System.IO.FileStream::.ctor [IP: 0010] ####
#### Test.Program::task1 [IP: 0096] ####

A first chance exception of type ‘System.OutOfMemoryException’ occurred in Microsoft.SPOT.IO.dll
A first chance exception of type ‘System.OutOfMemoryException’ occurred in System.IO.dll
An unhandled exception of type ‘System.OutOfMemoryException’ occurred in System.IO.dll

Code:



using System;
using System.Text;
using System.IO;
using System.Threading;
using Microsoft.SPOT;
using Microsoft.SPOT.IO;
using GHIElectronics.NETMF.IO;
using GHIElectronics.NETMF.USBHost;
using System.Xml;





namespace Test
{
class Program
{
    // Hold a static reference in case the GC kicks in and disposes it
// automatically, note that we only support one in this example!
static PersistentStorage ps;
static string rootDirectory;
 
static bool systemready;
static int hour;
static int minute;
static string str;
static string str1;
static string str2;

static FileStream stream;

public static void task1()
{
     hour =0;
     minute = 0; ;
    
     str1="";
     str2="";
  
    while (true)
    {
        Debug.Print("Task 1 is up");
        minute++;
        if (minute == 60)
        {
            minute=0;
            hour++;
        }
        if (hour == 24)
        {
            hour = 0;
            minute = 0;
        }
        str1 = @ "\sundata" + hour.ToString() + ".xml";
        str2 = "Pair" + minute.ToString();


        if (systemready)
        {


            stream = new FileStream(rootDirectory + str1, FileMode.Open);
            XmlReader reader = XmlReader.Create(stream);

            //read through all the nodes
            while (reader.Read())
            {
                //the headlines we want are in the item nodes
                if (reader.NodeType == XmlNodeType.Element)
                {
                    if (reader.Name == str2)
                    {
                        str="";
                        str = reader.GetAttribute(0);
                        Debug.Print(str);
                        str = reader.GetAttribute(1);
                        Debug.Print(str);
                        str = reader.GetAttribute(2);
                        Debug.Print(str);
                        str = reader.GetAttribute(3);
                        Debug.Print(str);
                    }
                }
            }
            stream.Close();
            stream.Dispose();
            reader.Close();
            reader.Dispose();
        }
        Debug.GC(true);
        Thread.Sleep(100);
        Debug.Print("Task 1 is down");  
         
    }
   
}



public static void Main()
{
    Debug.Print("this is main");    
    Thread task1Handler = new Thread(task1);
    task1Handler.Start();

// Subscribe to RemovableMedia events
RemovableMedia.Insert += RemovableMedia_Insert;
RemovableMedia.Eject += RemovableMedia_Eject;
// Subscribe to USB events
USBHostController.DeviceConnectedEvent += DeviceConnectedEvent;
// Sleep forever
Thread.Sleep(Timeout.Infinite);
}
static void DeviceConnectedEvent(USBH_Device device)
{
if (device.TYPE == USBH_DeviceType.MassStorage)
{
Debug.Print("USB Mass Storage detected...");
ps = new PersistentStorage(device);
ps.MountFileSystem();
}
}
static void RemovableMedia_Insert(object sender, MediaEventArgs e)
{
   

if (e.Volume.IsFormatted)
{


rootDirectory = VolumeInfo.GetVolumes()[0].RootDirectory;
systemready = true;









}
else
{
Debug.Print("Storage is not formatted. Format on PC with    FAT32/FAT16 first.");
}
}
static void RemovableMedia_Eject(object sender, MediaEventArgs e)
{
Debug.Print("Storage \"" + e.Volume.RootDirectory +
"\" is ejected.");
}
}
}
2

Add Debug.GC(true) in your loop, does it change anything?

You may want to post a small example code demonstrating the problem.

3

I didn’t see code in first post! Anyway, try what I suggested please

4 
    while (true)
    {
.....snip
        if (systemready)
        {
            stream = new FileStream(rootDirectory + str1, FileMode.Open);
....snip

Your creating a new filestream object with every single iteration through this loop. Eventually you’ll eat up all your memory.

5

Debug.GC(true) didn’t have any effect on the problem.

The reason why I open a new stream is because my data is spread over different files.

I thought using


  stream.Close();
  stream.Dispose();
  reader.Close();
  reader.Dispose();

Would remove those streams out of my memory?

6

You may have to open one steam globally and access it. Or make it null and force a collection before creating a new one.

Cheers Ian

7

Dispose should still release all used resources

8

Making the stream NULL and force collection also didnt have any effect.
I dont understand why dispose isnt working

9

Can you shrink it down to the smallest possible code we can try and then pass it on to us to try it out?

10

I tried to clean the code out as much as possible namely thread task1
Al the other code is namely for the usb support
I hope this helps

Here is the code for a small xml file sundata1.xml


<?xml version="1.0" encoding="utf-8" ?> <Data> <Pair1 Hour="0"  Minute="0 " Zenith="156.5553649846 " Azimuth="355.435421376177"></Pair1> <Pair2 Hour="0"  Minute="1 " Zenith="156.571160228868 " Azimuth="356.05862518525"></Pair2> <Pair3 Hour="0"  Minute="2 " Zenith="156.58460384937 " Azimuth="356.68254563175"></Pair3> </Data>

code


using System;
using System.Text;
using System.IO;
using System.Threading;
using Microsoft.SPOT;
using Microsoft.SPOT.IO;
using GHIElectronics.NETMF.IO;
using GHIElectronics.NETMF.USBHost;
using System.Xml;





namespace Test
{
    class Program
    {
        // Hold a static reference in case the GC kicks in and disposes it
        // automatically, note that we only support one in this example!
        static PersistentStorage ps;
        static string rootDirectory;
        static bool systemready;       
        static string str;       
        static FileStream stream;
        static XmlReader reader;
       

        public static void task1()
        {           

            while (true)
            {         
                
                if (systemready)
                {
                    stream = new FileStream(rootDirectory + @ "\sundata1.xml", FileMode.Open); ;
                    reader = XmlReader.Create(stream);
                                       

                    //read through all the nodes
                    while (reader.Read())
                    {
                        //the headlines we want are in the item nodes
                        if (reader.NodeType == XmlNodeType.Element)
                        {
                            if (reader.Name == "Pair2")
                            {
                                str = "";
                                str = reader.GetAttribute(0);
                                Debug.Print(str);
                                                               
                            }
                        }
                    }
                    stream.Close();
                    stream.Dispose();       
                                  
                    reader.Close();
                    reader.Dispose();
                    
                    
                   
                }
                Debug.GC(true);
                Thread.Sleep(100);
              

            }

        }



        public static void Main()
        {
            Debug.Print("this is main");
            Thread task1Handler = new Thread(task1);
            task1Handler.Start();

            // Subscribe to RemovableMedia events
            RemovableMedia.Insert += RemovableMedia_Insert;
            RemovableMedia.Eject += RemovableMedia_Eject;
            // Subscribe to USB events
            USBHostController.DeviceConnectedEvent += DeviceConnectedEvent;
            // Sleep forever
            Thread.Sleep(Timeout.Infinite);
        }
        static void DeviceConnectedEvent(USBH_Device device)
        {
            if (device.TYPE == USBH_DeviceType.MassStorage)
            {
                Debug.Print("USB Mass Storage detected...");
                ps = new PersistentStorage(device);
                ps.MountFileSystem();
            }
        }
        static void RemovableMedia_Insert(object sender, MediaEventArgs e)
        {


            if (e.Volume.IsFormatted)
            {
                rootDirectory = VolumeInfo.GetVolumes()[0].RootDirectory;         
 
                systemready = true;
            }
            else
            {
                Debug.Print("Storage is not formatted. Format on PC with    FAT32/FAT16 first.");
            }
        }
        static void RemovableMedia_Eject(object sender, MediaEventArgs e)
        {
            Debug.Print("Storage \"" + e.Volume.RootDirectory +
            "\" is ejected.");
        }
    }
}
11

Thank for reporting the issue. We opened an issue on codeplex and proposed a workaround:
http://netmf.codeplex.com/workitem/411

12

Thanks Mike for the workaround.
the memory leak is gone