Implement DataDownloader with empty constructor and add UTest (#8)

* feat: implement DataDownloader with empty constructor
feat: add UTest

* feat: add resolution when create new bar in file history provider
refactor: additional validation in Data Downloader

* refactor: neat log msg in LoadSymbol

* feat: Lazy initialization of IQFeedFileHistoryProvider

* fix: Improve IQFeed input data test for tick range between dates

* fix: path route in config file

* refactor: transform hardcoded port to logical use form

QuantConnect.IQFeed.Downloader/IQFeedDataDownloader.cs

@@ -15,26 +15,51 @@
 */
 
 using QuantConnect.Data;
+using QuantConnect.Logging;
 using QuantConnect.Securities;
 using QuantConnect.Data.Market;
+using QuantConnect.Configuration;
+using IQFeed.CSharpApiClient.Lookup;
 
 namespace QuantConnect.IQFeed.Downloader
 {
     /// <summary>
-    /// IQFeed Data Downloader class 
+    /// Represents a data downloader for retrieving historical market data using IQFeed.
     /// </summary>
     public class IQFeedDataDownloader : IDataDownloader
     {
-        private readonly IQFeedFileHistoryProvider _fileHistoryProvider;
-        private readonly TickType _tickType;
+        /// <summary>
+        /// The number of IQFeed clients to use for parallel processing.
+        /// </summary>
+        private const int NumberOfClients = 8;
+
+        /// <summary>
+        /// Lazy initialization for the IQFeed file history provider.
+        /// </summary>
+        /// <remarks>
+        /// This lazy initialization is used to provide deferred creation of the <see cref="IQFeedFileHistoryProvider"/>.
+        /// </remarks>
+        private Lazy<IQFeedFileHistoryProvider> _fileHistoryProviderLazy;
+
+        /// <summary>
+        /// The file history provider used by the data downloader.
+        /// </summary>
+        protected IQFeedFileHistoryProvider _fileHistoryProvider => _fileHistoryProviderLazy.Value;
 
         /// <summary>
-        /// Initializes a new instance of the <see cref="IQFeedDataDownloader"/> class
+        /// Initializes a new instance of the <see cref="IQFeedDataDownloader"/> class.
         /// </summary>
-        /// <param name="fileHistoryProvider"></param>
-        public IQFeedDataDownloader(IQFeedFileHistoryProvider fileHistoryProvider)
+        public IQFeedDataDownloader()
         {
-            _fileHistoryProvider = fileHistoryProvider;
+            _fileHistoryProviderLazy = new Lazy<IQFeedFileHistoryProvider>(() =>
+            {                
+                // Create and connect the IQFeed lookup client
+                var lookupClient = LookupClientFactory.CreateNew(Config.Get("iqfeed-host", "127.0.0.1"), IQSocket.GetPort(PortType.Lookup), NumberOfClients);
+                // Establish connection with IQFeed Client
+                lookupClient.Connect();
+
+                return new IQFeedFileHistoryProvider(lookupClient, new IQFeedDataQueueUniverseProvider(), MarketHoursDatabase.FromDataFolder());
+            });
         }
 
         /// <summary>
@@ -56,7 +81,15 @@ public IEnumerable<BaseData> Get(DataDownloaderGetParameters dataDownloaderGetPa
             }
 
             if (symbol.ID.SecurityType != SecurityType.Equity)
-                throw new NotSupportedException("SecurityType not available: " + symbol.ID.SecurityType);
+            {
+                return Enumerable.Empty<BaseData>();
+            }
+
+            if (tickType == TickType.Quote && resolution != Resolution.Tick)
+            {
+                Log.Trace($"{nameof(IQFeedDataDownloader)}.{nameof(Get)}: Historical data request with TickType 'Quote' is not supported for resolutions other than Tick. Requested Resolution: {resolution}");
+                return Enumerable.Empty<BaseData>();
+            }
 
             if (endUtc < startUtc)
                 throw new ArgumentException("The end date must be greater or equal than the start date.");

QuantConnect.IQFeed.Downloader/IQFeedDownloaderProgram.cs

@@ -20,7 +20,6 @@
 using IQFeed.CSharpApiClient;
 using QuantConnect.Securities;
 using QuantConnect.Configuration;
-using IQFeed.CSharpApiClient.Lookup;
 
 namespace QuantConnect.IQFeed.Downloader
 {
@@ -64,14 +63,9 @@ public static void IQFeedDownloader(IList<string> tickers, string resolution, Da
 
                 // Connect to IQFeed
                 IQFeedLauncher.Start(userName, password, productName, productVersion);
-                var lookupClient = LookupClientFactory.CreateNew(NumberOfClients);
-                lookupClient.Connect();
 
-                // Create IQFeed downloader instance
-                var universeProvider = new IQFeedDataQueueUniverseProvider();
-                var historyProvider = new IQFeedFileHistoryProvider(lookupClient, universeProvider, MarketHoursDatabase.FromDataFolder());
-                var downloader = new IQFeedDataDownloader(historyProvider);
-                var quoteDownloader = new IQFeedDataDownloader(historyProvider);
+                var downloader = new IQFeedDataDownloader();
+                var quoteDownloader = new IQFeedDataDownloader();
 
                 var resolutions = allResolution ? new List<Resolution> { Resolution.Tick, Resolution.Second, Resolution.Minute, Resolution.Hour, Resolution.Daily } : new List<Resolution> { castResolution };
                 var requests = resolutions.SelectMany(r => tickers.Select(t => new { Ticker = t, Resolution = r })).ToList();

QuantConnect.IQFeed.Tests/IQFeedDataDownloaderTest.cs

@@ -0,0 +1,58 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Logging;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.IQFeed.Downloader;
+
+namespace QuantConnect.IQFeed.Tests
+{
+    [TestFixture, Explicit("This tests require a IQFeed credentials.")]
+    public class IQFeedDataDownloaderTest
+    {
+        private IQFeedDataDownloader _downloader;
+
+        [SetUp]
+        public void SetUp()
+        {
+            _downloader = new IQFeedDataDownloader();
+        }
+
+        private static IEnumerable<TestCaseData> HistoricalDataTestCases => IQFeedHistoryProviderTests.HistoricalTestParameters;
+
+        [TestCaseSource(nameof(HistoricalDataTestCases))]
+        public void DownloadsHistoricalData(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool isEmptyResult)
+        {
+            var request = IQFeedHistoryProviderTests.CreateHistoryRequest(symbol, resolution, tickType, period);
+
+            var parameters = new DataDownloaderGetParameters(symbol, resolution, request.StartTimeUtc, request.EndTimeUtc, tickType);
+            var downloadResponse = _downloader.Get(parameters).ToList();
+
+            if (isEmptyResult)
+            {
+                Assert.IsEmpty(downloadResponse);
+                return;
+            }
+
+            IQFeedHistoryProviderTests.AssertHistoricalDataResponse(resolution, downloadResponse, isEmptyResult);
+        }
+    }
+}

QuantConnect.IQFeed.Tests/IQFeedHistoryProviderTests.cs

@@ -43,7 +43,7 @@ public void TearDown()
             _historyProvider.Dispose();
         }
 
-        private static IEnumerable<TestCaseData> HistoricalTestParameters
+        internal static IEnumerable<TestCaseData> HistoricalTestParameters
         {
             get
             {
@@ -84,6 +84,13 @@ public void GetHistoricalData(Symbol symbol, Resolution resolution, TickType tic
                 return;
             }
 
+            AssertTicksHaveAppropriateTickType(resolution, tickType, historyResponse);
+
+            AssertHistoricalDataResponse(resolution, historyResponse.SelectMany(x => x.AllData).ToList(), isEmptyResult);
+        }
+
+        internal static void AssertTicksHaveAppropriateTickType(Resolution resolution, TickType tickType, List<Slice> historyResponse)
+        {
             switch (resolution, tickType)
             {
                 case (Resolution.Tick, TickType.Quote):
@@ -93,11 +100,9 @@ public void GetHistoricalData(Symbol symbol, Resolution resolution, TickType tic
                     Assert.IsTrue(historyResponse.Any(x => x.Ticks.Any(xx => xx.Value.Count > 0 && xx.Value.Any(t => t.TickType == TickType.Trade))));
                     break;
             };
-
-            AssertHistoricalDataResponse(resolution, historyResponse.SelectMany(x => x.AllData).ToList(), isEmptyResult);
         }
 
-        private static void AssertHistoricalDataResponse(Resolution resolution, List<BaseData> historyResponse,  bool isEmptyResult)
+        internal static void AssertHistoricalDataResponse(Resolution resolution, List<BaseData> historyResponse,  bool isEmptyResult)
         {
             Assert.IsNotEmpty(historyResponse);
 
@@ -121,7 +126,7 @@ private static void AssertHistoricalDataResponse(Resolution resolution, List<Bas
 
         internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period)
         {
-            var end = new DateTime(2024, 01, 18, 12, 0, 0);
+            var end = new DateTime(2024, 01, 22, 12, 0, 0);
 
             if (resolution == Resolution.Daily)
             {

QuantConnect.IQFeed.Tests/QuantConnect.IQFeed.Tests.csproj

@@ -23,6 +23,7 @@
 	</ItemGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\..\Lean\Tests\QuantConnect.Tests.csproj" />
+		<ProjectReference Include="..\QuantConnect.IQFeed.Downloader\QuantConnect.IQFeed.Downloader.csproj" />
 		<ProjectReference Include="..\QuantConnect.IQFeed\QuantConnect.IQFeed.csproj" />
 	</ItemGroup>
 	<ItemGroup>

QuantConnect.IQFeed.Tests/config.json

@@ -1,4 +1,4 @@
 {
-  "data-folder": "../../../../../Lean/Data/",
-  "data-directory": "../../../../../Lean/Data/"
+  "data-folder": "../../../../Lean/Data/",
+  "data-directory": "../../../../Lean/Data/"
 }

QuantConnect.IQFeed/IQFeedDataQueueUniverseProvider.cs

@@ -268,13 +268,13 @@ private IEnumerable<SymbolData> LoadSymbols()
 
             if (mapExists)
             {
-                Log.Trace("Loading IQFeed futures symbol map file...");
+                Log.Trace($"{nameof(IQFeedDataQueueUniverseProvider)}.{nameof(LoadSymbols)}: Loading IQFeed futures symbol map file...");
                 _iqFeedNameMap = JsonConvert.DeserializeObject<Dictionary<string, string>>(File.ReadAllText(iqfeedNameMapFullName));
             }
 
             if (!universeExists)
             {
-                Log.Trace("Loading and unzipping IQFeed symbol universe file ({0})...", uri);
+                Log.Trace($"{nameof(IQFeedDataQueueUniverseProvider)}.{nameof(LoadSymbols)}: Loading and unzipping IQFeed symbol universe file ({0})...", uri);
 
                 using (var client = new WebClient())
                 {
@@ -286,7 +286,7 @@ private IEnumerable<SymbolData> LoadSymbols()
             }
             else
             {
-                Log.Trace("Found up-to-date IQFeed symbol universe file in local cache. Loading it...");
+                Log.Trace($"{nameof(IQFeedDataQueueUniverseProvider)}.{nameof(LoadSymbols)}: Found up-to-date IQFeed symbol universe file in local cache. Loading it...");
             }
 
             var symbolCache = new Dictionary<Symbol, SymbolData>();
@@ -303,7 +303,7 @@ private IEnumerable<SymbolData> LoadSymbols()
 
                 if (lineCounter % 100000 == 0)
                 {
-                    Log.Trace($"{lineCounter} lines read.");
+                    Log.Debug($"{nameof(IQFeedDataQueueUniverseProvider)}.{nameof(LoadSymbols)}: Processing file '{todayFullCsvName}': {lineCounter} lines read so far.");
                 }
 
                 prevPosition = currentPosition;
@@ -442,13 +442,13 @@ private IEnumerable<SymbolData> LoadSymbols()
 
             if (!mapExists)
             {
-                Log.Trace("Saving IQFeed futures symbol map file...");
+                Log.Trace($"{nameof(IQFeedDataQueueUniverseProvider)}.{nameof(LoadSymbols)}: IQFeed futures symbol map file not found. Saving a new map file...");
                 File.WriteAllText(iqfeedNameMapFullName, JsonConvert.SerializeObject(_iqFeedNameMap));
             }
 
             symbolUniverse.AddRange(symbolCache.Values);
 
-            Log.Trace("Finished loading IQFeed symbol universe file.");
+            Log.Debug($"{nameof(IQFeedDataQueueUniverseProvider)}.{nameof(LoadSymbols)}: Finished loading IQFeed symbol universe file.");
 
             return symbolUniverse;
         }

QuantConnect.IQFeed/IQFeedFileHistoryProvider.cs

@@ -191,7 +191,8 @@ private IEnumerable<BaseData> GetDataFromIntervalMessages(string filename, Histo
                     (decimal)interval.High,
                     (decimal)interval.Low,
                     (decimal)interval.Close,
-                    interval.PeriodVolume
+                    interval.PeriodVolume,
+                    request.Resolution.ToTimeSpan()
                 );
             }