Initialization via Job Packet and Rename DataQueueHandler to DataProvider

.github/workflows/build.yml

@@ -47,10 +47,10 @@ jobs:
           options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.IEX/Lean.DataSource.IEX -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }} -e QC_IEX_CLOUD_API_KEY=${{ secrets.QC_IEX_CLOUD_API_KEY }}
 
       - name: BuildDataSource
-        run: dotnet build ./QuantConnect.IEX/QuantConnect.IEX.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
+        run: dotnet build ./QuantConnect.IEX/QuantConnect.DataSource.IEX.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
 
       - name: BuildDataSourceTests
-        run: dotnet build ./QuantConnect.IEX.Tests/QuantConnect.IEX.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
+        run: dotnet build ./QuantConnect.IEX.Tests/QuantConnect.DataSource.IEX.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
 
       - name: Run Tests
-        run: dotnet test ./QuantConnect.IEX.Tests/bin/Release/QuantConnect.IEX.Tests.dll
+        run: dotnet test ./QuantConnect.IEX.Tests/bin/Release/QuantConnect.Lean.DataSource.IEX.Tests.dll

Lean.DataSource.IEX.sln

@@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
 VisualStudioVersion = 17.5.002.0
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.IEX.Tests", "QuantConnect.IEX.Tests\QuantConnect.IEX.Tests.csproj", "{38BD8123-9838-4984-A4D1-B8D1062197B4}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.DataSource.IEX.Tests", "QuantConnect.IEX.Tests\QuantConnect.DataSource.IEX.Tests.csproj", "{38BD8123-9838-4984-A4D1-B8D1062197B4}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.IEX", "QuantConnect.IEX\QuantConnect.IEX.csproj", "{6324C205-9693-4FA1-9EC1-9662E569D046}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.DataSource.IEX", "QuantConnect.IEX\QuantConnect.DataSource.IEX.csproj", "{6324C205-9693-4FA1-9EC1-9662E569D046}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.Tests", "..\Lean\Tests\QuantConnect.Tests.csproj", "{D6772C71-231E-44D1-9DD1-E796D393AAAB}"
 EndProject

QuantConnect.IEX.Tests/IEXDataDownloaderTests.cs

@@ -18,7 +18,7 @@
 using NUnit.Framework;
 using System.Collections.Generic;
 
-namespace QuantConnect.IEX.Tests
+namespace QuantConnect.Lean.DataSource.IEX.Tests
 {
     public class IEXDataDownloaderTests
     {

QuantConnect.IEX.Tests/IEXDataHistoryTests.cs

@@ -25,25 +25,25 @@
 using QuantConnect.Data.Market;
 using System.Collections.Generic;
 
-namespace QuantConnect.IEX.Tests
+namespace QuantConnect.Lean.DataSource.IEX.Tests
 {
     [TestFixture]
     public class IEXDataHistoryTests
     {
         private static MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
 
-        private IEXDataQueueHandler iexDataQueueHandler;
+        private IEXDataProvider iexDataProvider;
 
         [SetUp]
         public void SetUp()
         {
-            iexDataQueueHandler = new IEXDataQueueHandler();
+            iexDataProvider = new IEXDataProvider();
         }
 
         [TearDown]
         public void TearDown()
         {
-            iexDataQueueHandler.Dispose();
+            iexDataProvider.Dispose();
         }
 
         /// <summary>
@@ -228,8 +228,8 @@ private Slice[] GetHistory(Symbol symbol, Resolution resolution, TickType tickTy
         {
             var requests = new[] { CreateHistoryRequest(symbol, resolution, tickType, period) };
 
-            var slices = iexDataQueueHandler.GetHistory(requests, TimeZones.Utc).ToArray();
-            Log.Trace("Data points retrieved: " + iexDataQueueHandler.DataPointCount);
+            var slices = iexDataProvider.GetHistory(requests, TimeZones.Utc).ToArray();
+            Log.Trace("Data points retrieved: " + iexDataProvider.DataPointCount);
             Log.Trace("tick Type: " + tickType);
             return slices;
         }

QuantConnect.IEX.Tests/IEXDataQueueHandlerTests.cs → QuantConnect.IEX.Tests/IEXDataProviderTests.cs

@@ -19,6 +19,7 @@
 using System.Threading;
 using QuantConnect.Data;
 using QuantConnect.Tests;
+using QuantConnect.Packets;
 using QuantConnect.Logging;
 using Microsoft.CodeAnalysis;
 using System.Threading.Tasks;
@@ -28,13 +29,16 @@
 using System.Collections.Concurrent;
 using QuantConnect.Data.UniverseSelection;
 
-namespace QuantConnect.IEX.Tests
+namespace QuantConnect.Lean.DataSource.IEX.Tests
 {
     [TestFixture, Explicit("This tests require a iexcloud.io api key")]
-    public class IEXDataQueueHandlerTests
+    public class IEXDataProviderTests
     {
+        private readonly string _apiKey = Config.Get("iex-cloud-api-key");
+        private readonly string _pricePlan = Config.Get("iex-cloud-api-key");
+
         private CancellationTokenSource _cancellationTokenSource;
-        private IEXDataQueueHandler iexDataQueueHandler;
+        private IEXDataProvider iexDataProvider;
 
         private static readonly string[] HardCodedSymbolsSNP = {
             "AAPL", "MSFT", "AMZN", "FB", "GOOGL", "GOOG", "BRK.B", "JNJ", "PG", "NVDA", "V", "JPM", "HD", "UNH", "MA", "VZ",
@@ -76,15 +80,15 @@ public class IEXDataQueueHandlerTests
         public void SetUp()
         {
             _cancellationTokenSource = new();
-            iexDataQueueHandler = new IEXDataQueueHandler();
+            iexDataProvider = new IEXDataProvider();
         }
 
         [TearDown]
         public void TearDown()
         {
-            if (iexDataQueueHandler != null)
+            if (iexDataProvider != null)
             {
-                iexDataQueueHandler.Dispose();
+                iexDataProvider.Dispose();
             }
 
             _cancellationTokenSource.Dispose();
@@ -119,7 +123,7 @@ public void IEXCouldSubscribeManyTimes(Queue<SubscriptionDataConfig> configs)
             foreach (var config in configs)
             {
                 ProcessFeed(
-                    iexDataQueueHandler.Subscribe(config, (s, e) => { }),
+                    iexDataProvider.Subscribe(config, (s, e) => { }),
                     tick =>
                     {
                         if (tick != null)
@@ -155,7 +159,7 @@ public void IEXCouldSubscribeManyTimes(Queue<SubscriptionDataConfig> configs)
 
             foreach (var config in configs)
             {
-                iexDataQueueHandler.Unsubscribe(config);
+                iexDataProvider.Unsubscribe(config);
             }
 
             _cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(20));
@@ -177,12 +181,12 @@ public void IEXSubscribeToSeveralThenUnSubscribeExceptOne(Queue<SubscriptionData
             foreach (var config in configs)
             {
                 ProcessFeed(
-                    iexDataQueueHandler.Subscribe(config, (s, e) => { }),
+                    iexDataProvider.Subscribe(config, (s, e) => { }),
                     tick =>
                     {
                         if (tick != null)
                         {
-                            Log.Debug($"{nameof(IEXDataQueueHandlerTests)}: tick: {tick}");
+                            Log.Debug($"{nameof(IEXDataProviderTests)}: tick: {tick}");
 
                             tempDictionary.AddOrUpdate(tick.Symbol, 1, (id, count) => count + 1);
 
@@ -201,7 +205,7 @@ public void IEXSubscribeToSeveralThenUnSubscribeExceptOne(Queue<SubscriptionData
             for (int i = configs.Count; i > 2; i--)
             {
                 configs.TryDequeue(out var config);
-                iexDataQueueHandler.Unsubscribe(config);
+                iexDataProvider.Unsubscribe(config);
                 tempDictionary.TryRemove(config.Symbol, out _);
             }
 
@@ -216,7 +220,7 @@ public void IEXSubscribeToSeveralThenUnSubscribeExceptOne(Queue<SubscriptionData
             for (int i = configs.Count; i > 0; i--)
             {
                 configs.TryDequeue(out var config);
-                iexDataQueueHandler.Unsubscribe(config);
+                iexDataProvider.Unsubscribe(config);
             }
 
             Thread.Sleep(TimeSpan.FromSeconds(1));
@@ -244,7 +248,7 @@ public void IEXCouldSubscribeAndUnsubscribe()
             {
                 _cancellationTokenSource.Cancel();
             };
- 
+
             var configs = new[] {
                 GetSubscriptionDataConfig<TradeBar>(Symbol.Create("MBLY", SecurityType.Equity, Market.USA), Resolution.Second),
                 GetSubscriptionDataConfig<TradeBar>(Symbol.Create("USO", SecurityType.Equity, Market.USA), Resolution.Second)
@@ -253,14 +257,14 @@ public void IEXCouldSubscribeAndUnsubscribe()
             Array.ForEach(configs, (c) =>
             {
                 ProcessFeed(
-                    iexDataQueueHandler.Subscribe(c, (s, e) => { }),
+                    iexDataProvider.Subscribe(c, (s, e) => { }),
                     callback,
                     throwExceptionCallback);
             });
 
             Assert.IsFalse(_cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(20)), "The cancellation token was cancelled block thread.");
 
-            iexDataQueueHandler.Unsubscribe(Enumerable.First(configs, c => string.Equals(c.Symbol.Value, "MBLY")));
+            iexDataProvider.Unsubscribe(Enumerable.First(configs, c => string.Equals(c.Symbol.Value, "MBLY")));
 
             Log.Trace("Unsubscribing");
             _cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(2));
@@ -279,12 +283,12 @@ public void IEXCouldSubscribeMoreThan100Symbols()
             {
                 foreach (var config in GetSubscriptionDataConfigs(ticker, Resolution.Second))
                 {
-                    ProcessFeed(iexDataQueueHandler.Subscribe(config, (s, e) => { }), throwExceptionCallback: () => _cancellationTokenSource.Cancel());
+                    ProcessFeed(iexDataProvider.Subscribe(config, (s, e) => { }), throwExceptionCallback: () => _cancellationTokenSource.Cancel());
                 }
             }
 
             Assert.IsFalse(resetEvent.WaitOne(TimeSpan.FromMinutes(2), _cancellationTokenSource.Token), "The cancellation token was cancelled block thread.");
-            Assert.IsTrue(iexDataQueueHandler.IsConnected);
+            Assert.IsTrue(iexDataProvider.IsConnected);
         }
 
         [Test]
@@ -299,12 +303,12 @@ public void SubscribeOnALotOfSymbolsThrowArgumentExceptionExceedsAllowedLimit()
                 {
                     foreach (var config in GetSubscriptionDataConfigs(ticker, Resolution.Second))
                     {
-                        iexDataQueueHandler.Subscribe(config, (s, e) => { });
+                        iexDataProvider.Subscribe(config, (s, e) => { });
                     }
                 }
                 cancellationTokenSource.Token.WaitHandle.WaitOne(TimeSpan.FromSeconds(20));
             });
-            Assert.Less(iexDataQueueHandler.maxAllowedSymbolLimit, symbolCounter);
+            Assert.Less(iexDataProvider.maxAllowedSymbolLimit, symbolCounter);
         }
 
         [Test]
@@ -315,7 +319,7 @@ public void NotSubscribeOnUniverseSymbol()
 
             foreach (var config in GetSubscriptionDataConfigs(universeSymbol, Resolution.Second))
             {
-                Assert.IsNull(iexDataQueueHandler.Subscribe(config, (s, e) => { }));
+                Assert.IsNull(iexDataProvider.Subscribe(config, (s, e) => { }));
             }
         }
 
@@ -326,7 +330,7 @@ public void NotSubscribeOnCanonicalSymbol()
 
             foreach (var config in GetSubscriptionDataConfigs(spy, Resolution.Second))
             {
-                Assert.IsNull(iexDataQueueHandler.Subscribe(config, (s, e) => { }));
+                Assert.IsNull(iexDataProvider.Subscribe(config, (s, e) => { }));
             }
         }
 
@@ -337,20 +341,21 @@ public void SubscribeWithWrongApiKeyThrowException()
 
             var isSubscribeThrowException = false;
 
-            var iexDataQueueHandler = new IEXDataQueueHandler();
+            var iexDataProvider = new IEXDataProvider();
 
             foreach (var config in GetSubscriptionDataConfigs(Symbols.SPY, Resolution.Second))
             {
                 ProcessFeed(
-                    iexDataQueueHandler.Subscribe(config, (s, e) => { }),
+                    iexDataProvider.Subscribe(config, (s, e) => { }),
                     tick =>
                     {
-                        if (tick != null) 
+                        if (tick != null)
                         {
-                            Log.Debug($"{nameof(IEXDataQueueHandlerTests)}: tick: {tick}");
+                            Log.Debug($"{nameof(IEXDataProviderTests)}: tick: {tick}");
                         }
-                    }, 
-                    () => {
+                    },
+                    () =>
+                    {
                         isSubscribeThrowException = true;
                         _cancellationTokenSource.Cancel();
                     });
@@ -360,6 +365,60 @@ public void SubscribeWithWrongApiKeyThrowException()
             Assert.IsTrue(isSubscribeThrowException);
         }
 
+        [Test]
+        public void CanInitializeUsingJobPacket()
+        {
+            Config.Set("iex-cloud-api-key", "");
+
+            var job = new LiveNodePacket
+            {
+                BrokerageData = new Dictionary<string, string>() {
+                    { "iex-cloud-api-key", "InvalidApiKeyThatWontBeUsed" },
+                    { "iex-price-plan", "Launch" }
+                }
+            };
+
+            using var iexDataProvider = new IEXDataProvider();
+
+            Assert.Zero(iexDataProvider.maxAllowedSymbolLimit);
+
+            iexDataProvider.SetJob(job);
+
+            Assert.Greater(iexDataProvider.maxAllowedSymbolLimit, 0);
+
+            Config.Set("iex-cloud-api-key", _apiKey);
+        }
+
+        [Test]
+        public void JobPacketWontOverrideCredentials()
+        {
+            Config.Set("iex-cloud-api-key", "wrong_key");
+            Config.Set("iex-cloud-api-key", "Launch");
+
+            var job = new LiveNodePacket
+            {
+                BrokerageData = new Dictionary<string, string>() {
+                    { "iex-cloud-api-key", "InvalidApiKeyThatWontBeUsed" },
+                    { "iex-price-plan", "Enterprise" }
+                }
+            };
+
+            using var iexDataProvider = new IEXDataProvider();
+
+            var maxSymbolLimitBeforeSetJob = iexDataProvider.maxAllowedSymbolLimit;
+
+            // it has initialized already
+            Assert.Greater(maxSymbolLimitBeforeSetJob, 0);
+
+            iexDataProvider.SetJob(job);
+
+            // we use Enterprise plan in job variable => we must have unlimited maxAllowedSymbolLimit, but our config keep Launch
+            Assert.That(maxSymbolLimitBeforeSetJob, Is.EqualTo(iexDataProvider.maxAllowedSymbolLimit));
+
+            Config.Set("iex-cloud-api-key", _apiKey);
+            Config.Set("iex-cloud-api-key", _pricePlan);
+        }
+
         private void ProcessFeed(IEnumerator<BaseData> enumerator, Action<BaseData> callback = null, Action throwExceptionCallback = null)
         {
             Task.Factory.StartNew(() =>
@@ -385,7 +444,7 @@ private void ProcessFeed(IEnumerator<BaseData> enumerator, Action<BaseData> call
             }).ContinueWith(task =>
             {
                 if (throwExceptionCallback != null)
-                { 
+                {
                     throwExceptionCallback();
                 }
                 Log.Error("The throwExceptionCallback is null.");

QuantConnect.IEX.Tests/QuantConnect.IEX.Tests.csproj → QuantConnect.IEX.Tests/QuantConnect.DataSource.IEX.Tests.csproj

@@ -7,10 +7,10 @@
 		<IsPackable>false</IsPackable>
 		<TestProjectType>UnitTest</TestProjectType>
 		<OutputPath>bin\$(Configuration)\</OutputPath>
-		<Product>QuantConnect.IEX.Tests</Product>
-		<AssemblyName>QuantConnect.IEX.Tests</AssemblyName>
-		<RootNamespace>QuantConnect.IEX.Tests</RootNamespace>
-		<AssemblyTitle>QuantConnect.IEX.Tests</AssemblyTitle>
+		<Product>QuantConnect.Lean.DataSource.IEX.Tests</Product>
+		<AssemblyName>QuantConnect.Lean.DataSource.IEX.Tests</AssemblyName>
+		<RootNamespace>QuantConnect.Lean.DataSource.IEX.Tests</RootNamespace>
+		<AssemblyTitle>QuantConnect.Lean.DataSource.IEX.Tests</AssemblyTitle>
 		<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
 	</PropertyGroup>
 
@@ -26,7 +26,7 @@
 	</ItemGroup>
 
 	<ItemGroup>
-		<ProjectReference Include="..\QuantConnect.IEX\QuantConnect.IEX.csproj" />
+		<ProjectReference Include="..\QuantConnect.IEX\QuantConnect.DataSource.IEX.csproj" />
 		<ProjectReference Include="..\..\Lean\Tests\QuantConnect.Tests.csproj" />
 	</ItemGroup>
 

QuantConnect.IEX.Tests/TestSetup.cs

@@ -20,7 +20,7 @@
 using QuantConnect.Logging;
 using QuantConnect.Configuration;
 
-namespace QuantConnect.IEX.Tests
+namespace QuantConnect.Lean.DataSource.IEX.Tests
 {
     [SetUpFixture]
     public class TestSetup

QuantConnect.IEX/Constants/IEXDataStreamChannels.cs

@@ -14,7 +14,7 @@
  *
 */
 
-namespace QuantConnect.IEX.Constants
+namespace QuantConnect.Lean.DataSource.IEX.Constants
 {
     /// <summary>
     /// Represents constants for accessing IEX Cloud data streams channels, specifically related to last sale and top-of-book information.

QuantConnect.IEX/EnumeratorWrapper.cs

@@ -16,7 +16,7 @@
 using QuantConnect.Data;
 using System.Collections;
 
-namespace QuantConnect.IEX
+namespace QuantConnect.Lean.DataSource.IEX
 {
     /// <summary>
     /// The new enumerator wrapper for this subscription request