Further improvements

- Use Concurrent brokerage message handler - Handle cross zero holdings ordering - Fixes for clean up for historical API - Fix CI - Minor cleanup of IDataQieueUniverse
QuantConnect · Jul 17, 2024 · 7ffda22 · 7ffda22
1 parent 2cabd73
commit 7ffda22
Show file tree

Hide file tree

Showing 12 changed files with 383 additions and 436 deletions.
diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml
@@ -8,8 +8,13 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     env:
-        QC_TEMPLATE_BROKERAGE_KEY: ${{ secrets.QC_TEMPLATE_BROKERAGE_KEY }}
-        QC_TEMPLATE_BROKERAGE_SECRET: ${{ secrets.QC_TEMPLATE_BROKERAGE_SECRET }}
+      QC_ALPACA_API_KEY: ${{ secrets.QC_ALPACA_API_KEY }}
+      QC_ALPACA_API_SECRET: ${{ secrets.QC_ALPACA_API_SECRET }}
+      QC_ALPACA_ACCESS_TOKEN: ${{ secrets.QC_ALPACA_ACCESS_TOKEN }}
+      QC_ALPACA_PAPER_TRADING: ${{ secrets.QC_ALPACA_PAPER_TRADING }}
+      QC_JOB_USER_ID: ${{ secrets.QC_JOB_USER_ID }}
+      QC_API_ACCESS_TOKEN: ${{ secrets.QC_API_ACCESS_TOKEN }}
+      QC_JOB_ORGANIZATION_ID: ${{ secrets.QC_JOB_ORGANIZATION_ID }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -19,7 +24,7 @@ jobs:
       - uses: addnab/docker-run-action@v3
         with:
           image: quantconnect/lean:foundation
-          options: --workdir /__w/Lean.Brokerages.Template/Lean.Brokerages.Template -v /home/runner/work:/__w -e QC_TEMPLATE_BROKERAGE_KEY=${{ secrets.QC_TEMPLATE_BROKERAGE_KEY }} -e QC_TEMPLATE_BROKERAGE_SECRET=${{ secrets.QC_TEMPLATE_BROKERAGE_SECRET }} -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }}
+          options: --workdir /__w/Lean.Brokerages.Alpaca/Lean.Brokerages.Alpaca -v /home/runner/work:/__w -e QC_ALPACA_API_KEY=${{ secrets.QC_ALPACA_API_KEY }} -e QC_ALPACA_API_SECRET=${{ secrets.QC_ALPACA_API_SECRET }} -e QC_ALPACA_ACCESS_TOKEN=${{ secrets.QC_ALPACA_ACCESS_TOKEN }} -e QC_ALPACA_PAPER_TRADING=${{ secrets.QC_ALPACA_PAPER_TRADING }} -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }}
 
       - name: Checkout Lean Same Branch
         id: lean-same-branch
@@ -41,7 +46,7 @@ jobs:
         run: mv Lean ../Lean
 
       - name: Build
-        run: dotnet build /p:Configuration=Release /v:quiet /p:WarningLevel=1 QuantConnect.TemplateBrokerage.sln
+        run: dotnet build /p:Configuration=Release /v:quiet /p:WarningLevel=1 QuantConnect.AlpacaBrokerage.sln
 
       - name: Run Tests
-        run: dotnet test ./QuantConnect.TemplateBrokerage.Tests/bin/Release/QuantConnect.Brokerages.Template.Tests.dll
+        run: dotnet test ./QuantConnect.AlpacaBrokerage.Tests/bin/Release/QuantConnect.Brokerages.Alpaca.Tests.dll
diff --git a/QuantConnect.AlpacaBrokerage.Tests/AlpacaBrokerageAdditionalTests.cs b/QuantConnect.AlpacaBrokerage.Tests/AlpacaBrokerageAdditionalTests.cs
@@ -77,8 +77,8 @@ public TestAlpacaBrokerage(string apiKey, string apiKeySecret, bool isPaperTradi
             {
             }
 
-            public TestAlpacaBrokerage(string apiKey, string apiKeySecret, bool isPaperTrading, IOrderProvider orderProvider)
-                : base(apiKey, apiKeySecret, null, isPaperTrading, orderProvider)
+            public TestAlpacaBrokerage(string apiKey, string apiKeySecret, bool isPaperTrading, IOrderProvider orderProvider, ISecurityProvider securityProvider)
+                : base(apiKey, apiKeySecret, null, isPaperTrading, orderProvider, securityProvider)
             {
             }
 

diff --git a/QuantConnect.AlpacaBrokerage.Tests/AlpacaBrokerageHistoryProviderTests.cs b/QuantConnect.AlpacaBrokerage.Tests/AlpacaBrokerageHistoryProviderTests.cs
@@ -53,8 +53,6 @@ private static IEnumerable<TestCaseData> TestParameters
                 yield return new TestCaseData(Symbols.AAPL, Resolution.Hour, TickType.Quote, new DateTime(2024, 6, 17, 9, 30, 0), new DateTime(2024, 6, 17, 16, 0, 0));
                 yield return new TestCaseData(Symbols.AAPL, Resolution.Daily, TickType.Quote, new DateTime(2024, 6, 17, 9, 30, 0), new DateTime(2024, 6, 17, 16, 0, 0));
 
-                yield return new TestCaseData(Symbols.AAPL, Resolution.Tick, TickType.OpenInterest, new DateTime(2024, 6, 10, 9, 30, 0), new DateTime(2024, 6, 17, 16, 0, 0));
-
                 var AAPLOption = Symbol.CreateOption(Symbols.AAPL, Symbols.AAPL.ID.Market, OptionStyle.American, OptionRight.Call, 100, new DateTime(2024, 06, 21));
                 yield return new TestCaseData(AAPLOption, Resolution.Tick, TickType.Trade, new DateTime(2024, 6, 12, 9, 30, 0), new DateTime(2024, 6, 21, 16, 0, 0));
                 yield return new TestCaseData(AAPLOption, Resolution.Second, TickType.Trade, new DateTime(2024, 6, 12, 9, 30, 0), new DateTime(2024, 6, 21, 16, 0, 0));
@@ -88,6 +86,8 @@ private static IEnumerable<TestCaseData> NotSupportHistoryParameters
                 yield return new TestCaseData(Symbols.AAPL, Resolution.Hour, TickType.OpenInterest, new DateTime(default), new DateTime(default));
                 yield return new TestCaseData(Symbols.AAPL, Resolution.Daily, TickType.OpenInterest, new DateTime(default), new DateTime(default));
 
+                yield return new TestCaseData(Symbols.AAPL, Resolution.Tick, TickType.OpenInterest, new DateTime(2024, 6, 10, 9, 30, 0), new DateTime(2024, 6, 17, 16, 0, 0));
+
                 var AAPLOption = Symbol.CreateOption(Symbols.AAPL, Symbols.AAPL.ID.Market, OptionStyle.American, OptionRight.Call, 100, new DateTime(2024, 06, 21));
                 yield return new TestCaseData(AAPLOption, Resolution.Second, TickType.OpenInterest, new DateTime(default), new DateTime(default));
                 yield return new TestCaseData(AAPLOption, Resolution.Minute, TickType.OpenInterest, new DateTime(default), new DateTime(default));
@@ -101,7 +101,6 @@ private static IEnumerable<TestCaseData> NotSupportHistoryParameters
                 yield return new TestCaseData(AAPLOption, Resolution.Daily, TickType.Quote, new DateTime(2024, 6, 17, 9, 30, 0), new DateTime(2024, 6, 17, 16, 0, 0));
 
                 yield return new TestCaseData(Symbols.BTCUSD, Resolution.Daily, TickType.OpenInterest, new DateTime(default), new DateTime(default));
-                yield return new TestCaseData(Symbols.BTCUSD, Resolution.Daily, TickType.Quote, new DateTime(default), new DateTime(default));
             }
         }
 
@@ -120,6 +119,9 @@ public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType,
 
             var histories = _alpacaBrokerage.GetHistory(historyRequest).ToList();
             Assert.Greater(histories.Count, 0);
+            Assert.IsTrue(histories.All(x => x.EndTime - x.Time == resolution.ToTimeSpan()));
+            Assert.IsTrue(histories.All(x => x.Symbol == symbol));
+            Assert.IsTrue(histories.All(x => x is not Data.Market.Tick tick || tick.TickType == tickType));
         }
 
         internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTime,

diff --git a/QuantConnect.AlpacaBrokerage.Tests/AlpacaBrokerageTestHelpers.cs b/QuantConnect.AlpacaBrokerage.Tests/AlpacaBrokerageTestHelpers.cs
@@ -53,7 +53,7 @@ public static class AlpacaBrokerageTestHelpers
     /// </exception>
     public static (string ApiKey, string ApiKeySecret, bool IsPapperTrading) GetConfigParameters(bool isValidateOnEmpty = true)
     {
-        var (apiKey, apiKeySecret, isPaperTrading) = (Config.Get("alpaca-api-key-id"), Config.Get("alpaca-api-secret-key"), Config.GetBool("alpaca-use-paper-trading"));
+        var (apiKey, apiKeySecret, isPaperTrading) = (Config.Get("alpaca-api-key"), Config.Get("alpaca-api-secret"), Config.GetBool("alpaca-paper-trading"));
 
         if (!isValidateOnEmpty)
         {

diff --git a/QuantConnect.AlpacaBrokerage.Tests/AlpacaBrokerageTests.cs b/QuantConnect.AlpacaBrokerage.Tests/AlpacaBrokerageTests.cs
@@ -13,16 +13,16 @@
  * limitations under the License.
 */
 
-using System;
-using System.Linq;
 using NUnit.Framework;
-using QuantConnect.Tests;
-using QuantConnect.Orders;
-using QuantConnect.Logging;
 using QuantConnect.Interfaces;
+using QuantConnect.Logging;
+using QuantConnect.Orders;
 using QuantConnect.Securities;
-using System.Collections.Generic;
+using QuantConnect.Tests;
 using QuantConnect.Tests.Brokerages;
+using System;
+using System.Collections.Generic;
+using System.Linq;
 using static QuantConnect.Brokerages.Alpaca.Tests.AlpacaBrokerageAdditionalTests;
 
 namespace QuantConnect.Brokerages.Alpaca.Tests
@@ -37,15 +37,14 @@ protected override IBrokerage CreateBrokerage(IOrderProvider orderProvider, ISec
         {
             var (apiKey, apiKeySecret, isPaperTrading) = AlpacaBrokerageTestHelpers.GetConfigParameters();
 
-            return new TestAlpacaBrokerage(apiKey, apiKeySecret, isPaperTrading, orderProvider);
+            return new TestAlpacaBrokerage(apiKey, apiKeySecret, isPaperTrading, orderProvider, securityProvider);
         }
         protected override bool IsAsync() => false;
         protected override decimal GetAskPrice(Symbol symbol)
         {
             return (Brokerage as TestAlpacaBrokerage).GetLatestQuotePublic(symbol).AskPrice;
         }
 
-
         /// <summary>
         /// Provides the data required to test each order type in various cases
         /// </summary>
@@ -61,6 +60,22 @@ private static IEnumerable<TestCaseData> EquityOrderParameters
             }
         }
 
+        /// <summary>
+        /// Provides the data required to test each order type in various cases
+        /// </summary>
+        private static IEnumerable<TestCaseData> OptionOrderParameters
+        {
+            get
+            {
+
+                var option = Symbol.CreateOption(Symbols.AAPL, Symbols.AAPL.ID.Market, OptionStyle.American, OptionRight.Call, 230, new DateTime(2024, 12, 20));
+                yield return new TestCaseData(new MarketOrderTestParameters(option));
+                yield return new TestCaseData(new LimitOrderTestParameters(option, 20m, 10m));
+                yield return new TestCaseData(new StopMarketOrderTestParameters(option, 20m, 10m));
+                yield return new TestCaseData(new StopLimitOrderTestParameters(option, 20m, 10m));
+            }
+        }
+
         /// <summary>
         /// Provides the data required to test each order type in various cases
         /// </summary>
@@ -75,7 +90,7 @@ private static IEnumerable<TestCaseData> CryptoOrderParameters
             }
         }
 
-        [Test, TestCaseSource(nameof(EquityOrderParameters))]
+        [Test, TestCaseSource(nameof(OptionOrderParameters))]
         public override void CancelOrders(OrderTestParameters parameters)
         {
             base.CancelOrders(parameters);
@@ -87,6 +102,12 @@ public void CancelOrdersCrypto(OrderTestParameters parameters)
             base.CancelOrders(parameters);
         }
 
+        [Test, TestCaseSource(nameof(OptionOrderParameters))]
+        public void CancelOrdersOption(OrderTestParameters parameters)
+        {
+            base.CancelOrders(parameters);
+        }
+
         [Test, TestCaseSource(nameof(EquityOrderParameters))]
         public override void LongFromZero(OrderTestParameters parameters)
         {
@@ -99,12 +120,24 @@ public void LongFromZeroCrypto(OrderTestParameters parameters)
             base.LongFromZero(parameters);
         }
 
+        [Test, TestCaseSource(nameof(OptionOrderParameters))]
+        public void LongFromZeroOption(OrderTestParameters parameters)
+        {
+            base.LongFromZero(parameters);
+        }
+
         [Test, TestCaseSource(nameof(EquityOrderParameters))]
         public override void CloseFromLong(OrderTestParameters parameters)
         {
             base.CloseFromLong(parameters);
         }
 
+        [Test, TestCaseSource(nameof(OptionOrderParameters))]
+        public void CloseFromLongOption(OrderTestParameters parameters)
+        {
+            base.CloseFromLong(parameters);
+        }
+
         [Test, TestCaseSource(nameof(CryptoOrderParameters))]
         public void CloseFromLongCrypto(OrderTestParameters parameters)
         {
@@ -135,6 +168,12 @@ public override void ShortFromZero(OrderTestParameters parameters)
             base.ShortFromZero(parameters);
         }
 
+        [Test, TestCaseSource(nameof(OptionOrderParameters))]
+        public void ShortFromZeroOption(OrderTestParameters parameters)
+        {
+            base.ShortFromZero(parameters);
+        }
+
         [Test, TestCaseSource(nameof(CryptoOrderParameters))]
         public void ShortFromZeroCrypto(OrderTestParameters parameters)
         {
@@ -147,6 +186,12 @@ public override void CloseFromShort(OrderTestParameters parameters)
             base.CloseFromShort(parameters);
         }
 
+        [Test, TestCaseSource(nameof(OptionOrderParameters))]
+        public void CloseFromShortOption(OrderTestParameters parameters)
+        {
+            base.CloseFromShort(parameters);
+        }
+
         [Test, TestCaseSource(nameof(EquityOrderParameters))]
         [Explicit("Not supported: Different side position if we have bought 1 quantity we can not sell more then 1")]
         public override void ShortFromLong(OrderTestParameters parameters)

diff --git a/QuantConnect.AlpacaBrokerage.Tests/config.json b/QuantConnect.AlpacaBrokerage.Tests/config.json
@@ -1,7 +1,12 @@
 {
-  "data-folder": "../../../../Lean/Data/",
+    "data-folder": "../../../../Lean/Data/",
 
-  "alpaca-api-key-id": "",
-  "alpaca-api-secret-key": "",
-  "alpaca-use-paper-trading": true
-}
+    "job-user-id": "0",
+    "api-access-token": "",
+    "job-organization-id": "",
+
+    "alpaca-api-key": "",
+    "alpaca-api-secret": "",
+    "alpaca-access-token": "",
+    "alpaca-paper-trading": true
+}
diff --git a/QuantConnect.AlpacaBrokerage/AlpacaBrokerage.DataQueueHandler.cs b/QuantConnect.AlpacaBrokerage/AlpacaBrokerage.DataQueueHandler.cs
@@ -59,15 +59,15 @@ public void Unsubscribe(SubscriptionDataConfig dataConfig)
     public void SetJob(LiveNodePacket job)
     {
         // used for data
-        job.BrokerageData.TryGetValue("alpaca-api-key-id", out var apiKey);
-        job.BrokerageData.TryGetValue("alpaca-api-secret-key", out var secretKey);
+        job.BrokerageData.TryGetValue("alpaca-api-key", out var apiKey);
+        job.BrokerageData.TryGetValue("alpaca-api-secret", out var secretKey);
 
         // required for trading
         job.BrokerageData.TryGetValue("alpaca-access-token", out var accessToken);
 
-        var usePaperTrading = Convert.ToBoolean(job.BrokerageData["alpaca-use-paper-trading"]);
+        var usePaperTrading = Convert.ToBoolean(job.BrokerageData["alpaca-paper-trading"]);
 
-        Initialize(apiKey, secretKey, accessToken, usePaperTrading, null);
+        Initialize(apiKey, secretKey, accessToken, usePaperTrading, null, null);
         if (!IsConnected)
         {
             Connect();

diff --git a/QuantConnect.AlpacaBrokerage/AlpacaBrokerage.DataQueueUniverse.cs b/QuantConnect.AlpacaBrokerage/AlpacaBrokerage.DataQueueUniverse.cs
@@ -14,13 +14,11 @@
 */
 
 using System;
-using System.Linq;
 using Alpaca.Markets;
 using QuantConnect.Logging;
-using System.Threading.Tasks;
+using QuantConnect.Securities;
 using QuantConnect.Interfaces;
 using System.Collections.Generic;
-using System.Collections.Concurrent;
 
 namespace QuantConnect.Brokerages.Alpaca;
 
@@ -37,41 +35,26 @@ public IEnumerable<Symbol> LookupSymbols(Symbol symbol, bool includeExpired, str
     {
         if (!symbol.SecurityType.IsOption())
         {
-            Log.Error("The provided symbol is not an option. SecurityType: " + symbol.SecurityType);
-            return Enumerable.Empty<Symbol>();
+            Log.Error("AlpacaBrokerage.LookupSymbols(): The provided symbol is not an option. SecurityType: " + symbol.SecurityType);
+            yield break;
         }
-        var blockingOptionCollection = new BlockingCollection<Symbol>();
 
-        Task.Run(async () =>
-        {
-            var underlying = symbol.Underlying.Value;
-
-            var nextPageToken = default(string);
-            var optionContractRequest = new OptionContractsRequest(symbol.Underlying.Value) { ExpirationDateGreaterThanOrEqualTo = DateOnly.FromDateTime(DateTime.UtcNow) };
-            optionContractRequest.Pagination.Size = 10_000;
-            do
-            {
-                var response = await _tradingClient.ListOptionContractsAsync(optionContractRequest);
-                nextPageToken = response.NextPageToken;
-                foreach (var res in response.Items)
-                {
-                    blockingOptionCollection.Add(_symbolMapper.GetLeanSymbol(AssetClass.UsOption, res.Symbol));
-                }
-
-                optionContractRequest.Pagination.Token = nextPageToken;
-
-            } while (!string.IsNullOrEmpty(nextPageToken));
-        }).ContinueWith(_ => blockingOptionCollection.CompleteAdding());
+        var exchangeTimeZone = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
+        var exchangeDate = DateTime.UtcNow.ConvertFromUtc(exchangeTimeZone).Date;
 
-        var options = blockingOptionCollection.GetConsumingEnumerable();
-
-        // Validate if the collection contains at least one successful response from history.
-        if (!options.Any())
+        var nextPageToken = default(string);
+        var optionContractRequest = new OptionContractsRequest(symbol.Underlying.Value) { ExpirationDateGreaterThanOrEqualTo = DateOnly.FromDateTime(exchangeDate) };
+        optionContractRequest.Pagination.Size = 500;
+        do
         {
-            return null;
-        }
-
-        return options;
+            var response = _tradingClient.ListOptionContractsAsync(optionContractRequest).SynchronouslyAwaitTask();
+            nextPageToken = response.NextPageToken;
+            foreach (var res in response.Items)
+            {
+                yield return _symbolMapper.GetLeanSymbol(AssetClass.UsOption, res.Symbol);
+            }
+            optionContractRequest.Pagination.Token = nextPageToken;
+        } while (!string.IsNullOrEmpty(nextPageToken));
     }
 
     /// <summary>