Patricia
Published © CC BY-SA

Camera Stream From Device to PC

I use a proxy web application hosted inside Azure to transfer image data from my robot camera to the remote control app.

BeginnerProtip3 hours6,139
Camera Stream From Device to PC

Things used in this project

Hardware components

Raspberry Pi 2 Model B
Raspberry Pi 2 Model B
×1
Microsoft Lifecam 3000 USB Camera
You should try a different compatible camera if possible, this one turned out to have a very low capture rate.
×1
Old School PC :-)
×1

Software apps and online services

Microsoft Azure
Microsoft Azure
Windows 10 IoT Core
Microsoft Windows 10 IoT Core
Visual Studio 2015
Microsoft Visual Studio 2015

Story

Read more

Code

WebSocketCamera.cs

C#
Device class responsible for capturing camera data and sending it to webserver.
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading;
using System.Threading.Tasks;
using Windows.Media.Capture;
using Windows.Media.MediaProperties;
using Windows.Networking.Sockets;
using Windows.Storage.Streams;

namespace SimpleController.Domain
{
    public class WebSocketCamera
    {
        public class PutRequest
        {
            public string Image { get; set; }
        }

        private CancellationTokenSource mCancelaationToken;
        private LowLagPhotoCapture mLowLagCapture;
        private MediaCapture mMediaCapture;
        private DateTime mStartTime;
        private TimeSpan mAutoStopAfter = new TimeSpan(0, 5, 0);
        private StreamWebSocket mStreamWebSocket;

        private async Task init()
        {
            try
            {
                mMediaCapture = new MediaCapture();
                await mMediaCapture.InitializeAsync();
                mLowLagCapture = await mMediaCapture.PrepareLowLagPhotoCaptureAsync(ImageEncodingProperties.CreateJpeg());
                mStreamWebSocket = new StreamWebSocket();
                mStreamWebSocket.Closed += MStreamWebSocket_Closed;
            }
            catch (Exception ex)
            {
                AppInsights.Client.TrackException(ex);
            }
        }

        private void MStreamWebSocket_Closed(IWebSocket sender, WebSocketClosedEventArgs args)
        {
            closeSocket(sender);
        }

        public async Task StartCapture()
        {
            mCancelaationToken?.Cancel();
            if (mStreamWebSocket != null)
            {
                closeSocket(mStreamWebSocket);
            }

            await init();
            mCancelaationToken = new CancellationTokenSource();
            mStartTime = DateTime.UtcNow;

            try
            {
                await mStreamWebSocket.ConnectAsync(new Uri($"{Globals.WEBSOCKET_ENDPOINT}?device={MainPage.GetUniqueDeviceId()}"));

                var task = Task.Run(async () =>
                {
                    var socket = mStreamWebSocket;
                    while (!mCancelaationToken.IsCancellationRequested)
                    {
                        try
                        {
                            var capturedPhoto = await mLowLagCapture.CaptureAsync();
                            using (var rac = capturedPhoto.Frame.CloneStream())
                            {
                                var dr = new DataReader(rac.GetInputStreamAt(0));
                                var bytes = new byte[rac.Size];
                                await dr.LoadAsync((uint)rac.Size);
                                dr.ReadBytes(bytes);

                                await socket.OutputStream.WriteAsync(bytes.AsBuffer());
                            }
                        }
                        catch (Exception ex)
                        {
                            AppInsights.Client.TrackException(ex);
                        }

                        if ((DateTime.UtcNow - mStartTime) > mAutoStopAfter)
                        {
                            AppInsights.Client.TrackEvent("CameraAutoTurnOff");
                            mCancelaationToken.Cancel();
                        }
                    }
                }, mCancelaationToken.Token);
            }
            catch (Exception ex)
            {
                mStreamWebSocket.Dispose();
                mStreamWebSocket = null;
                AppInsights.Client.TrackException(ex);
            }

        }

        private void closeSocket(IWebSocket webSocket)
        {
            try
            {
                webSocket.Close(1000, "Closed due to user request.");
            }
            catch (Exception ex)
            {
                AppInsights.Client.TrackException(ex);
            }
        }

        private static async Task sendData(byte[] bytes)
        {
            try
            {
                HttpClient http = new HttpClient();
                var response = await http.PutAsJsonAsync<PutRequest>(new Uri("http://sgnexus.azurewebsites.net/api/imagedata/" + "f56bccc1-4075-d40f-7d0d-34c16a1411e0"), new PutRequest
                {
                    Image = Convert.ToBase64String(bytes)
                });

                var code = response.StatusCode;
                AppInsights.Client.TrackEvent("CameraDataSent", new Dictionary<string, string> { { "responsecode", code.ToString() } });
            }
            catch (Exception ex)
            {
                AppInsights.Client.TrackException(ex);
            }
        }

        public async Task StopCapture()
        {
            mCancelaationToken.Cancel();
            mCancelaationToken = null;
            mStreamWebSocket = null;
        }
    }
}

ImageSenderWebSocketMiddleware.cs

C#
Server-side class for receiving data from sender (device) and delegating it further to receiver (PC).
using Microsoft.AspNetCore.Http;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.WebSockets;
using System.Threading;
using System.Threading.Tasks;

namespace SGNexus
{
    public class ImageSenderWebSocketMiddleware
    {
        readonly RequestDelegate mNext;

        public ImageSenderWebSocketMiddleware(RequestDelegate next)
        {
            mNext = next;
        }

        public async Task Invoke(HttpContext http)
        {
            if (http.WebSockets.IsWebSocketRequest && http.Request.Query.ContainsKey("device"))
            {
                var deviceid = http.Request.Query["device"].ToString();
                var webSocket = await http.WebSockets.AcceptWebSocketAsync();
                if (webSocket.State == WebSocketState.Open)
                {
                    while (webSocket.State == WebSocketState.Open)
                    {
                        var buffer = new ArraySegment<Byte>(new Byte[4096]);
                        var received = await webSocket.ReceiveAsync(buffer, CancellationToken.None);

                        switch (received.MessageType)
                        {
                            case WebSocketMessageType.Close:
                                await webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closed in server by the client", CancellationToken.None);
                                continue;
                            case WebSocketMessageType.Binary:
                                List<byte> data = new List<byte>(buffer.Take(received.Count));
                                while (received.EndOfMessage == false)
                                {
                                    received = await webSocket.ReceiveAsync(buffer, CancellationToken.None);
                                    data.AddRange(buffer.Take(received.Count));
                                }

                                var socketconnectionList = ImageReceiverWebSocketMiddleware.Connections.Where(x => x.DeviceId.Equals(deviceid, StringComparison.Ordinal)).ToArray();

                                foreach (var socketconnection in socketconnectionList)
                                {
                                    var destsocket = socketconnection.SocketConnection;
                                    if (destsocket.State == System.Net.WebSockets.WebSocketState.Open)
                                    {
                                        var type = WebSocketMessageType.Binary;

                                        try
                                        {
                                            await destsocket.SendAsync(new ArraySegment<byte>(data.ToArray()), type, true, CancellationToken.None);
                                        }
                                        catch (Exception ex)
                                        {
                                            AppInsights.Client.TrackException(ex);
                                        }
                                    }
                                    else
                                    {
                                        AppInsights.Client.TrackTrace("Removing closed connection");
                                        ImageReceiverWebSocketMiddleware.Connections.Remove(socketconnection);
                                    }
                                }

                                break;
                        }
                    }
                }
            }
            else
            {
                await mNext.Invoke(http);
            }
        }

        public class SocketConnections
        {
            public string DeviceId { get; set; }
            public WebSocket SocketConnection { get; set; }
        }
    }
}

ImageReceiverWebSocketMiddleware.cs

C#
Server-side class responsible for accepting PC connections. Stores them in a public list.
using Microsoft.AspNetCore.Http;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.WebSockets;
using System.Threading;
using System.Threading.Tasks;

namespace SGNexus
{
    public class ImageReceiverWebSocketMiddleware
    {
        public static List<SocketConnections> Connections { get; set; }

        readonly RequestDelegate mNext;

        static ImageReceiverWebSocketMiddleware()
        {
            Connections = new List<SocketConnections>();
        }

        public ImageReceiverWebSocketMiddleware(RequestDelegate next)
        {
            mNext = next;
        }

        public async Task Invoke(HttpContext http)
        {
            if (http.WebSockets.IsWebSocketRequest && http.Request.Query.ContainsKey("device"))
            {
                var deviceid = http.Request.Query["device"].ToString();
                var webSocket = await http.WebSockets.AcceptWebSocketAsync();
                if (webSocket.State == WebSocketState.Open)
                {
                    var existigsocketconnection = ImageReceiverWebSocketMiddleware.Connections.Where(x => x.DeviceId.Equals(deviceid)).FirstOrDefault();
                    if (existigsocketconnection != null)
                    {
                        ImageReceiverWebSocketMiddleware.Connections.Remove(existigsocketconnection);
                    }
                    Connections.Add(new SocketConnections { DeviceId = deviceid, SocketConnection = webSocket });
                    while (webSocket.State == WebSocketState.Open)
                    {
                        var buffer = new ArraySegment<Byte>(new Byte[4096]);
                        var received = await webSocket.ReceiveAsync(buffer, CancellationToken.None);

                        switch (received.MessageType)
                        {
                            case WebSocketMessageType.Close:
                                var socket = Connections.Where(x => x.SocketConnection == webSocket).First();
                                Connections.Remove(socket);
                                await webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closed in server by the client", CancellationToken.None);
                                continue;
                        }
                    }
                }
            }
            else
            {
                await mNext.Invoke(http);
            }
        }

        public class SocketConnections
        {
            public string DeviceId { get; set; }
            public WebSocket SocketConnection { get; set; }
        }
    }
}

EventsReaderViewModel.cs

C#
PC-Application class that connects to and receives data from webserver.
using GalaSoft.MvvmLight;
using Microsoft.ServiceBus.Messaging;
using RemoteControl.Wpf.Model;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Media.Imaging;

namespace RemoteControl.Wpf.ViewModel
{
    public class EventsReaderViewModel : ViewModelBase
    {
        public delegate void BitmapAquiredEventHandler(object sender, BitmapSource e);
        public event BitmapAquiredEventHandler BitmapAquired;

        static string iotHubD2cEndpoint = "messages/events";
        static EventHubClient eventHubClient;

        public EventsReaderViewModel()
        {
            DeviceId = Globals.DEVICE_ID;

            Log = new ObservableCollection<DataItem>();

            Console.WriteLine("Receive messages.\n");
            eventHubClient = EventHubClient.CreateFromConnectionString(Globals.CONNECTIONSTRING_OWNER, iotHubD2cEndpoint);

            var d2cPartitions = eventHubClient.GetRuntimeInformation().PartitionIds;

            CancellationTokenSource cts = new CancellationTokenSource();

            System.Console.CancelKeyPress += (s, e) =>
            {
                e.Cancel = true;
                cts.Cancel();
                Console.WriteLine("Exiting...");
            };

            var tasks = new List<Task>();
            foreach (string partition in d2cPartitions)
            {
                tasks.Add(receiveMessagesFromDeviceAsync(partition, cts.Token));
            }
            tasks.Add(receiveWebSocketMessagesFromDeviceAsync(cts.Token));
        }

        private string mDeviceId;

        public string DeviceId
        {
            get { return mDeviceId; }
            set { Set(ref mDeviceId, value); }
        }

        private ObservableCollection<DataItem> mLog;

        public ObservableCollection<DataItem> Log
        {
            get { return mLog; }
            set { Set(ref mLog, value); }
        }

        private async Task receiveMessagesFromDeviceAsync(string partition, CancellationToken ct)
        {
            try
            {
                eventHubClient.GetDefaultConsumerGroup().Abort();
                await eventHubClient.GetDefaultConsumerGroup().CloseAsync();
                var eventHubReceiver = eventHubClient.GetDefaultConsumerGroup().CreateReceiver(partition, DateTime.UtcNow);
                while (true)
                {
                    try
                    {
                        if (ct.IsCancellationRequested) break;
                        EventData eventData = await eventHubReceiver.ReceiveAsync();
                        if (eventData == null) continue;

                        string data = Encoding.UTF8.GetString(eventData.GetBytes());
                        if (eventData.Properties.ContainsKey("path"))
                        {
                            var path = eventData.Properties["path"];
                            if (String.Equals(path.ToString(), "imagefeed", StringComparison.InvariantCultureIgnoreCase))
                            {
                                var dataBytes = eventData.GetBytes();
                                MemoryStream ms = new MemoryStream(dataBytes, 0, dataBytes.Length);
                                var image = Image.FromStream(ms);
                                var oldBitmap = new Bitmap(image);
                                var bitmapSource = System.Windows.Interop.Imaging.CreateBitmapSourceFromHBitmap(
                                          oldBitmap.GetHbitmap(System.Drawing.Color.Transparent),
                                          IntPtr.Zero,
                                          new Int32Rect(0, 0, oldBitmap.Width, oldBitmap.Height),
                                          null);

                                var del = BitmapAquired;
                                if (del != null)
                                {
                                    del(this, bitmapSource);
                                }
                                var picturespath = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);
                                image.Save(Path.Combine(picturespath, "lastimagefromrover.jpp"));
                                addToLog(string.Format("Image message received"));
                            }
                        }
                        else
                        {
                            addToLog(string.Format("Message received. Partition: {0} Data: '{1}'", partition, data));
                        }
                    }
                    catch (Exception ex)
                    {
                        addToLog(ex.Message.ToString());
                    }

                }
            }
            catch (Exception ex)
            {
                addToLog(ex.Message.ToString());
            }
        }

        private async Task receiveWebSocketMessagesFromDeviceAsync(CancellationToken ct)
        {
            try
            {
                string wsUri = $"{Globals.WEBSOCKET_ENDPOINT}?device={Globals.DEVICE_ID}";
                var socket = new ClientWebSocket();
                await socket.ConnectAsync(new Uri(wsUri), ct);

                while (socket.State == WebSocketState.Open)
                {
                    try
                    {
                        if (ct.IsCancellationRequested) break;

                        var buffer = new ArraySegment<Byte>(new Byte[40960]);
                        WebSocketReceiveResult rcvResult = await socket.ReceiveAsync(buffer, ct);
                        string b64 = String.Empty;
                        if (rcvResult.MessageType == WebSocketMessageType.Binary)
                        {
                            List<byte> data = new List<byte>(buffer.Take(rcvResult.Count));
                            while (rcvResult.EndOfMessage == false)
                            {
                                rcvResult = await socket.ReceiveAsync(buffer, CancellationToken.None);
                                data.AddRange(buffer.Take(rcvResult.Count));
                            }

                            MemoryStream ms = new MemoryStream(data.ToArray());

                            var image = Image.FromStream(ms);
                            var oldBitmap = new Bitmap(image);
                            var bitmapSource = System.Windows.Interop.Imaging.CreateBitmapSourceFromHBitmap(
                                        oldBitmap.GetHbitmap(System.Drawing.Color.Transparent),
                                        IntPtr.Zero,
                                        new Int32Rect(0, 0, oldBitmap.Width, oldBitmap.Height),
                                        null);

                            var del = BitmapAquired;
                            if (del != null)
                            {
                                del(this, bitmapSource);
                            }
                            var picturespath = Environment.GetFolderPath(Environment.SpecialFolder.MyPictures);
                            image.Save(Path.Combine(picturespath, "lastimagefromrover.jpg"));
                            addToLog(string.Format("Image message received"));
                        }
                    }
                    catch (Exception ex)
                    {
                        addToLog(ex.Message.ToString());
                    }
                }
            }
            catch (Exception ex)
            {
                addToLog(ex.Message.ToString());
            }
        }

        private void addToLog(string message)
        {
            if (Log == null)
            {
                Log = new ObservableCollection<DataItem>();
            }
            Log.Insert(0, new DataItem(message));
            if (Log.Count > 100)
            {
                Log.RemoveAt(100);
            }
        }
    }
}

Full source code for Pi Device / WebApp / Remote Control

StargateSuperRobot folder contains full source code.

Credits

Patricia

Patricia

5 projects • 18 followers
A c# developer. As child i used to play with electronics and this source of fun is living up again in me, as we enter the age of IoT.

Comments