From 89db062e5ca99339b1ed2ca1ca9ef555f898433f Mon Sep 17 00:00:00 2001 From: Roman Yavnikov <45608740+Romazes@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:01:00 +0200 Subject: [PATCH] Migrate CoinApi Integration to Standalone Project (#1) * feat: remove template files * feat: prepare solution's structure feat: github issue & PR templates feat: prepare GH workflow file * feat: paste coinApi files from ToolBox * feat: paste coinApi.Converter files from ToolBox * feat: paste coinApi.test files from ToolBox feat: TestSetup initializer * feat: divide HistoryProvider in file * remove: does not support exchanges in SymbolMapper * refactor: downgrade pckg Microsoft.NET.Test.Sdk * feat: handle JsonSerializationException feat: simplify code feat: remove one warning msg * feat: DQH tests with different param feat: GetBrokerage CryptoFuture Symbol Test feat: init with wrong api key test feat: helper class for tests * feat: forceTypeNameOnExisting set up to false for GetExportedValueByTypeName * feat: add sync bash script in Converter project * remove: unsupported markets from Converter * feat: ValidateSubscription * fix: productId in ValidateSubscription * refactor: make static of TestHelper class fix: deprecated GDAX to Coinbase Market in Symbol test * feat: CoinAPIDataDownloader fea: test of CoinAPIDataDownloader fix: reset config in wrong api key test * fix: handle exception when parsing response in History * feat: update Readme * Create LICENSE * fix: reset config in test where we change config * refactor: create RestClient at once time * refactor: create RestRequest only once * rename: Converter to DataProcessing * refactor: test OneTimeSetUp to testing class * refactor: increase delay in DQH tests * fix: change delay and init DQH class tests * refactor: GlobalSetup make static * refactor: ProcessFeed in DQH tests * feat: add some explicit and log trace * refactor: validation on null tick remove: thread sleep * feat: add delay in ProcessFeed by cancellationToken refactor: future test * fix: tick symbol in CryptoFuture test * remove: Explicit attribute in tests --- .github/ISSUE_TEMPLATE.md | 33 ++ .github/PULL_REQUEST_TEMPLATE.md | 31 ++ .github/workflows/build.yml | 35 +- .gitignore | 1 + DataProcessing/CLRImports.py | 12 - DataProcessing/CoinApiDataConverter.cs | 255 +++++++++ DataProcessing/CoinApiDataConverterProgram.cs | 38 ++ DataProcessing/CoinApiDataReader.cs | 189 +++++++ DataProcessing/CoinApiEntryData.cs | 44 ++ DataProcessing/DataProcessing.csproj | 71 ++- DataProcessing/MyCustomDataDownloader.cs | 236 -------- DataProcessing/Program.cs | 91 +-- DataProcessing/config.json | 5 - DataProcessing/process.sample.ipynb | 78 --- DataProcessing/process.sample.py | 32 -- DataProcessing/process.sample.sh | 0 DataProcessing/sync.sh | 40 ++ Demonstration.cs | 77 --- Demonstration.py | 47 -- DemonstrationUniverse.cs | 66 --- DemonstrationUniverse.py | 50 -- DropboxDownloader.py | 119 ---- LICENSE | 201 +++++++ Lean.DataSource.CoinAPI.sln | 37 ++ MyCustomDataType.cs | 156 ------ MyCustomDataUniverseType.cs | 141 ----- .../CoinAPIDataDownloaderTests.cs | 110 ++++ .../CoinAPIHistoryProviderTests.cs | 126 +++++ .../CoinAPISymbolMapperTests.cs | 79 +++ .../CoinApiAdditionalTests.cs | 38 ++ .../CoinApiDataQueueHandlerTest.cs | 235 ++++++++ .../CoinApiTestHelper.cs | 100 ++++ .../QuantConnect.CoinAPI.Tests.csproj | 42 ++ QuantConnect.CoinAPI.Tests/TestSetup.cs | 64 +++ QuantConnect.CoinAPI.Tests/config.json | 7 + QuantConnect.CoinAPI/CoinAPIDataDownloader.cs | 78 +++ .../CoinApi.HistoryProvider.cs | 169 ++++++ .../CoinApiDataQueueHandler.cs | 526 ++++++++++++++++++ QuantConnect.CoinAPI/CoinApiProduct.cs | 49 ++ QuantConnect.CoinAPI/CoinApiSymbol.cs | 42 ++ QuantConnect.CoinAPI/CoinApiSymbolMapper.cs | 266 +++++++++ QuantConnect.CoinAPI/Messages/BaseMessage.cs | 25 + QuantConnect.CoinAPI/Messages/ErrorMessage.cs | 25 + QuantConnect.CoinAPI/Messages/HelloMessage.cs | 40 ++ .../Messages/HistoricalDataMessage.cs | 46 ++ QuantConnect.CoinAPI/Messages/QuoteMessage.cs | 46 ++ QuantConnect.CoinAPI/Messages/TradeMessage.cs | 46 ++ .../Models/CoinApiErrorResponse.cs | 31 ++ .../QuantConnect.CoinAPI.csproj | 40 ++ QuantConnect.DataSource.csproj | 27 - README.md | 81 +-- examples.md | 1 - output/alternative/mycustomdatatype/spy.csv | 6 - renameDataset.sh | 57 -- tests/MyCustomDataTypeTests.cs | 99 ---- tests/Tests.csproj | 23 - 56 files changed, 3236 insertions(+), 1373 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md delete mode 100644 DataProcessing/CLRImports.py create mode 100644 DataProcessing/CoinApiDataConverter.cs create mode 100644 DataProcessing/CoinApiDataConverterProgram.cs create mode 100644 DataProcessing/CoinApiDataReader.cs create mode 100644 DataProcessing/CoinApiEntryData.cs delete mode 100644 DataProcessing/MyCustomDataDownloader.cs delete mode 100644 DataProcessing/config.json delete mode 100644 DataProcessing/process.sample.ipynb delete mode 100644 DataProcessing/process.sample.py delete mode 100644 DataProcessing/process.sample.sh create mode 100644 DataProcessing/sync.sh delete mode 100644 Demonstration.cs delete mode 100644 Demonstration.py delete mode 100644 DemonstrationUniverse.cs delete mode 100644 DemonstrationUniverse.py delete mode 100644 DropboxDownloader.py create mode 100644 LICENSE create mode 100644 Lean.DataSource.CoinAPI.sln delete mode 100644 MyCustomDataType.cs delete mode 100644 MyCustomDataUniverseType.cs create mode 100644 QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs create mode 100644 QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs create mode 100644 QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs create mode 100644 QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs create mode 100644 QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs create mode 100644 QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs create mode 100644 QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj create mode 100644 QuantConnect.CoinAPI.Tests/TestSetup.cs create mode 100644 QuantConnect.CoinAPI.Tests/config.json create mode 100644 QuantConnect.CoinAPI/CoinAPIDataDownloader.cs create mode 100644 QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs create mode 100644 QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs create mode 100644 QuantConnect.CoinAPI/CoinApiProduct.cs create mode 100644 QuantConnect.CoinAPI/CoinApiSymbol.cs create mode 100644 QuantConnect.CoinAPI/CoinApiSymbolMapper.cs create mode 100644 QuantConnect.CoinAPI/Messages/BaseMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/ErrorMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/HelloMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/QuoteMessage.cs create mode 100644 QuantConnect.CoinAPI/Messages/TradeMessage.cs create mode 100644 QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs create mode 100644 QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj delete mode 100644 QuantConnect.DataSource.csproj delete mode 100644 examples.md delete mode 100644 output/alternative/mycustomdatatype/spy.csv delete mode 100644 renameDataset.sh delete mode 100644 tests/MyCustomDataTypeTests.cs delete mode 100644 tests/Tests.csproj diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..b1be05d --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,33 @@ + + +## Expected Behavior + + + +## Current Behavior + + + +## Possible Solution + + + +## Steps to Reproduce (for bugs) + + +1. +2. +3. +4. + +## Context + + + +## Your Environment + +* Version used: +* Environment name and version (e.g. PHP 5.4 on nginx 1.9.1): +* Server type and version: +* Operating System and version: +* Link to your project: diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..4f9ceac --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,31 @@ + + +## Description + + +## Motivation and Context + + + +## How Has This Been Tested? + + + + +## Screenshots (if appropriate): + +## Types of changes + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to change) + +## Checklist: + + +- [ ] My code follows the code style of this project. +- [ ] My change requires a change to the documentation. +- [ ] I have updated the documentation accordingly. +- [ ] I have read the **CONTRIBUTING** document. +- [ ] I have added tests to cover my changes. +- [ ] All new and existing tests passed. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 09449fc..1a93820 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,18 +9,18 @@ on: jobs: build: runs-on: ubuntu-20.04 - + env: + QC_JOB_USER_ID: ${{ secrets.QC_JOB_USER_ID }} + QC_API_ACCESS_TOKEN: ${{ secrets.QC_API_ACCESS_TOKEN }} + QC_JOB_ORGANIZATION_ID: ${{ secrets.QC_JOB_ORGANIZATION_ID }} + QC_COINAPI_API_KEY: ${{ secrets.QC_COINAPI_API_KEY }} steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 - name: Free space run: df -h && rm -rf /opt/hostedtoolcache* && df -h - - name: Pull Foundation Image - uses: addnab/docker-run-action@v3 - with: - image: quantconnect/lean:foundation - - name: Checkout Lean Same Branch id: lean-same-branch uses: actions/checkout@v2 @@ -40,11 +40,20 @@ jobs: - name: Move Lean run: mv Lean ../Lean - - name: BuildDataSource - run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + - name: Pull Foundation Image + uses: addnab/docker-run-action@v3 + with: + image: quantconnect/lean:foundation + options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.CoinAPI/Lean.DataSource.CoinAPI -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }} -e QC_COINAPI_API_KEY=${{ secrets.QC_COINAPI_API_KEY }} + + - name: Build QuantConnect.CoinAPI + run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + + - name: Build DataProcessing + run: dotnet build ./DataProcessing/DataProcessing.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - name: BuildTests - run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 + - name: Build QuantConnect.CoinAPI.Tests + run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1 - - name: Run Tests - run: dotnet test ./tests/bin/Release/net6.0/Tests.dll + - name: Run QuantConnect.CoinAPI.Tests + run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.CoinAPI.Tests.dll \ No newline at end of file diff --git a/.gitignore b/.gitignore index 866dd07..7cd3cc7 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ *.hex # QC Cloud Setup Bash Files +!DataProcessing/*.sh *.sh # Include docker launch scripts for Mac/Linux !run_docker.sh diff --git a/DataProcessing/CLRImports.py b/DataProcessing/CLRImports.py deleted file mode 100644 index fca9342..0000000 --- a/DataProcessing/CLRImports.py +++ /dev/null @@ -1,12 +0,0 @@ -# This file is used to import the environment and classes/methods of LEAN. -# so that any python file could be using LEAN's classes/methods. -from clr_loader import get_coreclr -from pythonnet import set_runtime - -# process.runtimeconfig.json is created when we build the DataProcessing Project: -# dotnet build .\DataProcessing\DataProcessing.csproj -set_runtime(get_coreclr('process.runtimeconfig.json')) - -from AlgorithmImports import * -from QuantConnect.Lean.Engine.DataFeeds import * -AddReference("Fasterflect") \ No newline at end of file diff --git a/DataProcessing/CoinApiDataConverter.cs b/DataProcessing/CoinApiDataConverter.cs new file mode 100644 index 0000000..ccd00e9 --- /dev/null +++ b/DataProcessing/CoinApiDataConverter.cs @@ -0,0 +1,255 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Data; +using QuantConnect.Util; +using System.Diagnostics; +using QuantConnect.Logging; +using QuantConnect.ToolBox; +using QuantConnect.CoinAPI; + +namespace QuantConnect.DataProcessing +{ + /// + /// Console application for converting CoinApi raw data into Lean data format for high resolutions (tick, second and minute) + /// + public class CoinApiDataConverter + { + /// + /// List of supported exchanges + /// + private static readonly HashSet SupportedMarkets = new[] + { + Market.Coinbase, + Market.Bitfinex, + Market.Binance, + Market.Kraken, + Market.BinanceUS + }.ToHashSet(); + + private readonly DirectoryInfo _rawDataFolder; + private readonly DirectoryInfo _destinationFolder; + private readonly SecurityType _securityType; + private readonly DateTime _processingDate; + private readonly string _market; + + /// + /// CoinAPI data converter. + /// + /// the processing date. + /// path to the raw data folder. + /// destination of the newly generated files. + /// The security type to process + /// The market to process (optional). Defaults to processing all markets in parallel. + public CoinApiDataConverter(DateTime date, string rawDataFolder, string destinationFolder, string market = null, SecurityType securityType = SecurityType.Crypto) + { + _market = string.IsNullOrWhiteSpace(market) + ? null + : market.ToLowerInvariant(); + + _processingDate = date; + _securityType = securityType; + _rawDataFolder = new DirectoryInfo(Path.Combine(rawDataFolder, "crypto", "coinapi")); + if (!_rawDataFolder.Exists) + { + throw new ArgumentException($"CoinApiDataConverter(): Source folder not found: {_rawDataFolder.FullName}"); + } + + _destinationFolder = new DirectoryInfo(destinationFolder); + _destinationFolder.Create(); + } + + /// + /// Runs this instance. + /// + /// + public bool Run() + { + var stopwatch = Stopwatch.StartNew(); + + var symbolMapper = new CoinApiSymbolMapper(); + var success = true; + + // There were cases of files with with an extra suffix, following pattern: + // --_SPOT___.csv.gz + // Those cases should be ignored for SPOT prices. + var tradesFolder = new DirectoryInfo( + Path.Combine( + _rawDataFolder.FullName, + "trades", + _processingDate.ToStringInvariant(DateFormat.EightCharacter))); + + var quotesFolder = new DirectoryInfo( + Path.Combine( + _rawDataFolder.FullName, + "quotes", + _processingDate.ToStringInvariant(DateFormat.EightCharacter))); + + var rawMarket = _market != null && + CoinApiSymbolMapper.MapMarketsToExchangeIds.TryGetValue(_market, out var rawMarketValue) + ? rawMarketValue + : null; + + var securityTypeFilter = (string name) => name.Contains("_SPOT_"); + if (_securityType == SecurityType.CryptoFuture) + { + securityTypeFilter = (string name) => name.Contains("_FTS_") || name.Contains("_PERP_"); + } + + // Distinct by tick type and first two parts of the raw file name, separated by '-'. + // This prevents us from double processing the same ticker twice, in case we're given + // two raw data files for the same symbol. Related: https://github.com/QuantConnect/Lean/pull/3262 + var apiDataReader = new CoinApiDataReader(symbolMapper); + var filesToProcessCandidates = tradesFolder.EnumerateFiles("*.gz") + .Concat(quotesFolder.EnumerateFiles("*.gz")) + .Where(f => securityTypeFilter(f.Name) && (rawMarket == null || f.Name.Contains(rawMarket))) + .Where(f => f.Name.Split('_').Length == 4) + .ToList(); + + var filesToProcessKeys = new HashSet(); + var filesToProcess = new List(); + + foreach (var candidate in filesToProcessCandidates) + { + try + { + var entryData = apiDataReader.GetCoinApiEntryData(candidate, _processingDate, _securityType); + CurrencyPairUtil.DecomposeCurrencyPair(entryData.Symbol, out var baseCurrency, out var quoteCurrency); + + if (!candidate.FullName.Contains(baseCurrency) && !candidate.FullName.Contains(quoteCurrency)) + { + throw new Exception($"Skipping {candidate.FullName} we have the wrong symbol {entryData.Symbol}!"); + } + + var key = candidate.Directory.Parent.Name + entryData.Symbol.ID; + if (filesToProcessKeys.Add(key)) + { + // Separate list from HashSet to preserve ordering of viable candidates + filesToProcess.Add(candidate); + } + } + catch (Exception err) + { + // Most likely the exchange isn't supported. Log exception message to avoid excessive stack trace spamming in console output + Log.Error(err.Message); + } + } + + Parallel.ForEach(filesToProcess, (file, loopState) => + { + Log.Trace($"CoinApiDataConverter(): Starting data conversion from source file: {file.Name}..."); + try + { + ProcessEntry(apiDataReader, file); + } + catch (Exception e) + { + Log.Error(e, $"CoinApiDataConverter(): Error processing entry: {file.Name}"); + success = false; + loopState.Break(); + } + } + ); + + Log.Trace($"CoinApiDataConverter(): Finished in {stopwatch.Elapsed}"); + return success; + } + + /// + /// Processes the entry. + /// + /// The coinapi data reader. + /// The file. + private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file) + { + var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate, _securityType); + + if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market)) + { + // only convert data for supported exchanges + return; + } + + var tickData = coinapiDataReader.ProcessCoinApiEntry(entryData, file); + + // in some cases the first data points from '_processingDate' get's included in the previous date file + // so we will ready previous date data and drop most of it just to save these midnight ticks + var yesterdayDate = _processingDate.AddDays(-1); + var yesterdaysFile = new FileInfo(file.FullName.Replace( + _processingDate.ToStringInvariant(DateFormat.EightCharacter), + yesterdayDate.ToStringInvariant(DateFormat.EightCharacter))); + if (yesterdaysFile.Exists) + { + var yesterdaysEntryData = coinapiDataReader.GetCoinApiEntryData(yesterdaysFile, yesterdayDate, _securityType); + tickData = tickData.Concat(coinapiDataReader.ProcessCoinApiEntry(yesterdaysEntryData, yesterdaysFile)); + } + else + { + Log.Error($"CoinApiDataConverter(): yesterdays data file not found '{yesterdaysFile.FullName}'"); + } + + // materialize the enumerable into a list, since we need to enumerate over it twice + var ticks = tickData.Where(tick => tick.Time.Date == _processingDate) + .OrderBy(t => t.Time) + .ToList(); + + var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); + writer.Write(ticks); + + Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}"); + var consolidators = new List(); + + if (entryData.TickType == TickType.Trade) + { + consolidators.AddRange(new[] + { + new TradeTickAggregator(Resolution.Second), + new TradeTickAggregator(Resolution.Minute) + }); + } + else + { + consolidators.AddRange(new[] + { + new QuoteTickAggregator(Resolution.Second), + new QuoteTickAggregator(Resolution.Minute) + }); + } + + foreach (var tick in ticks) + { + if (tick.Suspicious) + { + // When CoinAPI loses connectivity to the exchange, they indicate + // it in the data by providing a value of `-1` for bid/ask price. + // We will keep it in tick data, but will remove it from consolidated data. + continue; + } + + foreach (var consolidator in consolidators) + { + consolidator.Update(tick); + } + } + + foreach (var consolidator in consolidators) + { + writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); + writer.Write(consolidator.Flush()); + } + } + } +} diff --git a/DataProcessing/CoinApiDataConverterProgram.cs b/DataProcessing/CoinApiDataConverterProgram.cs new file mode 100644 index 0000000..0609b17 --- /dev/null +++ b/DataProcessing/CoinApiDataConverterProgram.cs @@ -0,0 +1,38 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Globalization; + +namespace QuantConnect.DataProcessing +{ + /// + /// Coin API Main entry point for ToolBox. + /// + public static class CoinApiDataConverterProgram + { + public static void CoinApiDataProgram(string date, string rawDataFolder, string destinationFolder, string market, string securityType) + { + var processingDate = DateTime.ParseExact(date, DateFormat.EightCharacter, CultureInfo.InvariantCulture); + var typeToProcess = SecurityType.Crypto; + if (!string.IsNullOrEmpty(securityType)) + { + typeToProcess = (SecurityType)Enum.Parse(typeof(SecurityType), securityType, true); + } + var converter = new CoinApiDataConverter(processingDate, rawDataFolder, destinationFolder, market, typeToProcess); + converter.Run(); + } + } +} diff --git a/DataProcessing/CoinApiDataReader.cs b/DataProcessing/CoinApiDataReader.cs new file mode 100644 index 0000000..420dec6 --- /dev/null +++ b/DataProcessing/CoinApiDataReader.cs @@ -0,0 +1,189 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Globalization; +using QuantConnect.Logging; +using System.IO.Compression; +using QuantConnect.Brokerages; +using QuantConnect.Data.Market; +using CompressionMode = System.IO.Compression.CompressionMode; + +namespace QuantConnect.DataProcessing +{ + /// + /// Reader class for CoinAPI crypto raw data. + /// + public class CoinApiDataReader + { + private readonly ISymbolMapper _symbolMapper; + + /// + /// Creates a new instance of the class + /// + /// The symbol mapper + public CoinApiDataReader(ISymbolMapper symbolMapper) + { + _symbolMapper = symbolMapper; + } + + /// + /// Gets the coin API entry data. + /// + /// The source file. + /// The processing date. + /// The security type of this file. + /// + public CoinApiEntryData GetCoinApiEntryData(FileInfo file, DateTime processingDate, SecurityType securityType) + { + // crypto///-563-BITFINEX_SPOT_BTC_USD.csv.gz + var tickType = file.FullName.Contains("trades", StringComparison.InvariantCultureIgnoreCase) ? TickType.Trade : TickType.Quote; + + var symbolId = Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(file.Name)).Split('-').Last(); + + var symbol = _symbolMapper.GetLeanSymbol(symbolId, securityType, null); + + return new CoinApiEntryData + { + Name = file.Name, + Symbol = symbol, + TickType = tickType, + Date = processingDate + }; + } + + /// + /// Gets an list of ticks for a given CoinAPI source file. + /// + /// The entry data. + /// The file. + /// + /// CoinApiDataReader.ProcessCoinApiEntry(): CSV header not found for entry name: {entryData.Name} + public IEnumerable ProcessCoinApiEntry(CoinApiEntryData entryData, FileInfo file) + { + Log.Trace("CoinApiDataReader.ProcessTarEntry(): Processing " + + $"{entryData.Symbol.ID.Market}-{entryData.Symbol.Value}-{entryData.TickType}-{entryData.Symbol.SecurityType} " + + $"for {entryData.Date:yyyy-MM-dd}"); + + + using (var stream = new GZipStream(file.OpenRead(), CompressionMode.Decompress)) + using (var reader = new StreamReader(stream)) + { + var headerLine = reader.ReadLine(); + if (headerLine == null) + { + throw new Exception($"CoinApiDataReader.ProcessCoinApiEntry(): CSV header not found for entry name: {entryData.Name}"); + } + + var headerParts = headerLine.Split(';').ToList(); + + var tickList = entryData.TickType == TickType.Trade + ? ParseTradeData(entryData.Symbol, reader, headerParts) + : ParseQuoteData(entryData.Symbol, reader, headerParts); + + foreach (var tick in tickList) + { + yield return tick; + } + } + } + + /// + /// Parses CoinAPI trade data. + /// + /// The symbol. + /// The reader. + /// The header parts. + /// + private IEnumerable ParseTradeData(Symbol symbol, StreamReader reader, List headerParts) + { + var columnTime = headerParts.FindIndex(x => x == "time_exchange"); + var columnPrice = headerParts.FindIndex(x => x == "price"); + var columnQuantity = headerParts.FindIndex(x => x == "base_amount"); + + string line; + while ((line = reader.ReadLine()) != null) + { + var lineParts = line.Split(';'); + + var time = DateTime.Parse(lineParts[columnTime], CultureInfo.InvariantCulture); + var price = lineParts[columnPrice].ToDecimal(); + var quantity = lineParts[columnQuantity].ToDecimal(); + + yield return new Tick + { + Symbol = symbol, + Time = time, + Value = price, + Quantity = quantity, + TickType = TickType.Trade, + Suspicious = price == -1 + }; + } + } + + /// + /// Parses CoinAPI quote data. + /// + /// The symbol. + /// The reader. + /// The header parts. + /// + private IEnumerable ParseQuoteData(Symbol symbol, StreamReader reader, List headerParts) + { + var columnTime = headerParts.FindIndex(x => x == "time_exchange"); + var columnAskPrice = headerParts.FindIndex(x => x == "ask_px"); + var columnAskSize = headerParts.FindIndex(x => x == "ask_sx"); + var columnBidPrice = headerParts.FindIndex(x => x == "bid_px"); + var columnBidSize = headerParts.FindIndex(x => x == "bid_sx"); + + var previousAskPrice = 0m; + var previousBidPrice = 0m; + + string line; + while ((line = reader.ReadLine()) != null) + { + var lineParts = line.Split(';'); + + var time = DateTime.Parse(lineParts[columnTime], CultureInfo.InvariantCulture); + var askPrice = lineParts[columnAskPrice].ToDecimal(); + var askSize = lineParts[columnAskSize].ToDecimal(); + var bidPrice = lineParts[columnBidPrice].ToDecimal(); + var bidSize = lineParts[columnBidSize].ToDecimal(); + + if (askPrice == previousAskPrice && bidPrice == previousBidPrice) + { + // only save quote if bid price or ask price changed + continue; + } + + previousAskPrice = askPrice; + previousBidPrice = bidPrice; + + yield return new Tick + { + Symbol = symbol, + Time = time, + AskPrice = askPrice, + AskSize = askSize, + BidPrice = bidPrice, + BidSize = bidSize, + TickType = TickType.Quote, + Suspicious = bidPrice == -1m || askPrice == -1m + }; + } + } + } +} diff --git a/DataProcessing/CoinApiEntryData.cs b/DataProcessing/CoinApiEntryData.cs new file mode 100644 index 0000000..a4d5eb0 --- /dev/null +++ b/DataProcessing/CoinApiEntryData.cs @@ -0,0 +1,44 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.DataProcessing +{ + /// + /// Contains information extracted from CoinAPI entry name + /// + public class CoinApiEntryData + { + /// + /// The entry name + /// + public string Name { get; set; } + + /// + /// The LEAN symbol + /// + public Symbol Symbol { get; set; } + + /// + /// The tick type (Trade or Quote) + /// + public TickType TickType { get; set; } + + /// + /// The date of the entry + /// + public DateTime Date { get; set; } + } +} \ No newline at end of file diff --git a/DataProcessing/DataProcessing.csproj b/DataProcessing/DataProcessing.csproj index 26b5466..a5bc5cb 100644 --- a/DataProcessing/DataProcessing.csproj +++ b/DataProcessing/DataProcessing.csproj @@ -1,30 +1,41 @@ - - - Exe - net6.0 - process - true - - - - - - - - - - - - - - - PreserveNewest - - - - - - PreserveNewest - - - \ No newline at end of file + + + + Release + AnyCPU + Exe + net6.0 + process + true + bin\$(Configuration) + false + QuantConnect LEAN CoinAPI Converter Data Source: CoinAPI Converter Data Source plugin for Lean + enable + enable + + + + full + bin\Debug\ + + + + pdbonly + bin\Release\ + + + + + + + + + + + + + PreserveNewest + + + + diff --git a/DataProcessing/MyCustomDataDownloader.cs b/DataProcessing/MyCustomDataDownloader.cs deleted file mode 100644 index 1bfef32..0000000 --- a/DataProcessing/MyCustomDataDownloader.cs +++ /dev/null @@ -1,236 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json; -using QuantConnect.Configuration; -using QuantConnect.Data.Auxiliary; -using QuantConnect.DataSource; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Logging; -using QuantConnect.Util; - -namespace QuantConnect.DataProcessing -{ - /// - /// MyCustomDataDownloader implementation. - /// - public class MyCustomDataDownloader : IDisposable - { - public const string VendorName = "VendorName"; - public const string VendorDataName = "VendorDataName"; - - private readonly string _destinationFolder; - private readonly string _universeFolder; - private readonly string _clientKey; - private readonly string _dataFolder = Globals.DataFolder; - private readonly bool _canCreateUniverseFiles; - private readonly int _maxRetries = 5; - private static readonly List _defunctDelimiters = new() - { - '-', - '_' - }; - private ConcurrentDictionary> _tempData = new(); - - private readonly JsonSerializerSettings _jsonSerializerSettings = new() - { - DateTimeZoneHandling = DateTimeZoneHandling.Utc - }; - - /// - /// Control the rate of download per unit of time. - /// - private readonly RateGate _indexGate; - - /// - /// Creates a new instance of - /// - /// The folder where the data will be saved - /// The Vendor API key - public MyCustomDataDownloader(string destinationFolder, string apiKey = null) - { - _destinationFolder = Path.Combine(destinationFolder, VendorDataName); - _universeFolder = Path.Combine(_destinationFolder, "universe"); - _clientKey = apiKey ?? Config.Get("vendor-auth-token"); - _canCreateUniverseFiles = Directory.Exists(Path.Combine(_dataFolder, "equity", "usa", "map_files")); - - // Represents rate limits of 10 requests per 1.1 second - _indexGate = new RateGate(10, TimeSpan.FromSeconds(1.1)); - - Directory.CreateDirectory(_destinationFolder); - Directory.CreateDirectory(_universeFolder); - } - - /// - /// Runs the instance of the object. - /// - /// True if process all downloads successfully - public bool Run() - { - var stopwatch = Stopwatch.StartNew(); - var today = DateTime.UtcNow.Date; - - throw new NotImplementedException(); - - Log.Trace($"MyCustomDataDownloader.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}"); - return true; - } - - /// - /// Sends a GET request for the provided URL - /// - /// URL to send GET request for - /// Content as string - /// Failed to get data after exceeding retries - private async Task HttpRequester(string url) - { - for (var retries = 1; retries <= _maxRetries; retries++) - { - try - { - using (var client = new HttpClient()) - { - client.BaseAddress = new Uri(""); - client.DefaultRequestHeaders.Clear(); - - // You must supply your API key in the HTTP header, - // otherwise you will receive a 403 Forbidden response - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Token", _clientKey); - - // Responses are in JSON: you need to specify the HTTP header Accept: application/json - client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - - // Makes sure we don't overrun Quiver rate limits accidentally - _indexGate.WaitToProceed(); - - var response = await client.GetAsync(Uri.EscapeUriString(url)); - if (response.StatusCode == HttpStatusCode.NotFound) - { - Log.Error($"MyCustomDataDownloader.HttpRequester(): Files not found at url: {Uri.EscapeUriString(url)}"); - response.DisposeSafely(); - return string.Empty; - } - - if (response.StatusCode == HttpStatusCode.Unauthorized) - { - var finalRequestUri = response.RequestMessage.RequestUri; // contains the final location after following the redirect. - response = client.GetAsync(finalRequestUri).Result; // Reissue the request. The DefaultRequestHeaders configured on the client will be used, so we don't have to set them again. - } - - response.EnsureSuccessStatusCode(); - - var result = await response.Content.ReadAsStringAsync(); - response.DisposeSafely(); - - return result; - } - } - catch (Exception e) - { - Log.Error(e, $"MyCustomDataDownloader.HttpRequester(): Error at HttpRequester. (retry {retries}/{_maxRetries})"); - Thread.Sleep(1000); - } - } - - throw new Exception($"Request failed with no more retries remaining (retry {_maxRetries}/{_maxRetries})"); - } - - /// - /// Saves contents to disk, deleting existing zip files - /// - /// Final destination of the data - /// file name - /// Contents to write - private void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) - { - name = name.ToLowerInvariant(); - var finalPath = Path.Combine(destinationFolder, $"{name}.csv"); - var finalFileExists = File.Exists(finalPath); - - var lines = new HashSet(contents); - if (finalFileExists) - { - foreach (var line in File.ReadAllLines(finalPath)) - { - lines.Add(line); - } - } - - var finalLines = destinationFolder.Contains("universe") ? - lines.OrderBy(x => x.Split(',').First()).ToList() : - lines - .OrderBy(x => DateTime.ParseExact(x.Split(',').First(), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal)) - .ToList(); - - var tempPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.tmp"); - File.WriteAllLines(tempPath, finalLines); - var tempFilePath = new FileInfo(tempPath); - tempFilePath.MoveTo(finalPath, true); - } - - /// - /// Tries to normalize a potentially defunct ticker into a normal ticker. - /// - /// Ticker as received from Estimize - /// Set as the non-defunct ticker - /// true for success, false for failure - private static bool TryNormalizeDefunctTicker(string ticker, out string nonDefunctTicker) - { - // The "defunct" indicator can be in any capitalization/case - if (ticker.IndexOf("defunct", StringComparison.OrdinalIgnoreCase) > 0) - { - foreach (var delimChar in _defunctDelimiters) - { - var length = ticker.IndexOf(delimChar); - - // Continue until we exhaust all delimiters - if (length == -1) - { - continue; - } - - nonDefunctTicker = ticker[..length].Trim(); - return true; - } - - nonDefunctTicker = string.Empty; - return false; - } - - nonDefunctTicker = ticker; - return true; - } - - /// - /// Disposes of unmanaged resources - /// - public void Dispose() - { - _indexGate?.Dispose(); - } - } -} \ No newline at end of file diff --git a/DataProcessing/Program.cs b/DataProcessing/Program.cs index b203eb3..5dd4f5c 100644 --- a/DataProcessing/Program.cs +++ b/DataProcessing/Program.cs @@ -1,80 +1,43 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.IO; +using QuantConnect.Logging; +using System.Globalization; using QuantConnect.Configuration; -using QuantConnect.Logging; -using QuantConnect.Util; namespace QuantConnect.DataProcessing { - /// - /// Entrypoint for the data downloader/converter - /// - public class Program + internal class Program { - /// - /// Entrypoint of the program - /// - /// Exit code. 0 equals successful, and any other value indicates the downloader/converter failed. - public static void Main() + public static string DataFleetDeploymentDate = "QC_DATAFLEET_DEPLOYMENT_DATE"; + + private static void Main(string[] args) { - // Get the config values first before running. These values are set for us - // automatically to the value set on the website when defining this data type - var destinationDirectory = Path.Combine( - Config.Get("temp-output-directory", "/temp-output-directory"), - "alternative", - "vendorname"); + var rawDataFolder = new DirectoryInfo(Config.Get("raw-data-folder", "/raw")); + var temporaryOutputDirectory = Config.Get("temp-output-directory", "/temp-output-directory"); + // Allow for caller to specify which market they want to process + var market = args.Length != 0 && !string.IsNullOrWhiteSpace(args[0]) + ? args[0] + : null; - MyCustomDataDownloader instance = null; - try + var securityType = SecurityType.Crypto; + if (args.Length > 1 && !string.IsNullOrWhiteSpace(args[1])) { - // Pass in the values we got from the configuration into the downloader/converter. - instance = new MyCustomDataDownloader(destinationDirectory); - } - catch (Exception err) - { - Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {MyCustomDataDownloader.VendorDataName} {MyCustomDataDownloader.VendorDataName} data failed to be constructed"); - Environment.Exit(1); + securityType = (SecurityType)Enum.Parse(typeof(SecurityType), args[1], true); } - // No need to edit anything below here for most use cases. - // The downloader/converter is ran and cleaned up for you safely here. - try - { - // Run the data downloader/converter. - var success = instance.Run(); - if (!success) - { - Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process {MyCustomDataDownloader.VendorName} {MyCustomDataDownloader.VendorDataName} data"); - Environment.Exit(1); - } - } - catch (Exception err) + Environment.SetEnvironmentVariable(DataFleetDeploymentDate, "20240211"); + + var processingDateValue = Environment.GetEnvironmentVariable(DataFleetDeploymentDate); + var processingDate = DateTime.ParseExact(processingDateValue, "yyyyMMdd", CultureInfo.InvariantCulture); + + Log.Trace($"Price.Crypto.CoinApi.Main(): Processing {processingDate} for market: {market ?? "*"} SecurityType: {securityType}"); + + var converter = new CoinApiDataConverter(processingDate, rawDataFolder.FullName, temporaryOutputDirectory, market, securityType); + + if (!converter.Run()) { - Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {MyCustomDataDownloader.VendorDataName} {MyCustomDataDownloader.VendorDataName} data exited unexpectedly"); + Log.Error($"Price.Crypto.CoinApi.Main(): Processing CoinAPI for date {processingDate:yyyy-MM-dd} failed"); Environment.Exit(1); } - finally - { - // Run cleanup of the downloader/converter once it has finished or crashed. - instance.DisposeSafely(); - } - - // The downloader/converter was successful + Environment.Exit(0); } } diff --git a/DataProcessing/config.json b/DataProcessing/config.json deleted file mode 100644 index 1d1e2f2..0000000 --- a/DataProcessing/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "data-folder": "../../../Data/", - - "vendor-auth-token": "" -} \ No newline at end of file diff --git a/DataProcessing/process.sample.ipynb b/DataProcessing/process.sample.ipynb deleted file mode 100644 index 2493379..0000000 --- a/DataProcessing/process.sample.ipynb +++ /dev/null @@ -1,78 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "9b8eae46", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# CLRImports is required to handle Lean C# objects for Mapped Datasets (Single asset and Universe Selection)\n", - "# Requirements:\n", - "# python -m pip install clr-loader==0.1.7\n", - "# python -m pip install pythonnet==3.0.0a2\n", - "# This script must be executed in ./bin/Debug/net6.0 after the follwing command is executed\n", - "# dotnet build .\\DataProcessing\\\n", - "import os\n", - "from CLRImports import *\n", - "\n", - "# To use QuantBook, we need to set its internal handlers\n", - "# We download LEAN confif with the default settings \n", - "with open(\"quantbook.json\", 'w') as fp:\n", - " from requests import get\n", - " response = get(\"https://raw.githubusercontent.com/QuantConnect/Lean/master/Launcher/config.json\")\n", - " fp.write(response.text)\n", - "\n", - "Config.SetConfigurationFile(\"quantbook.json\")\n", - "Config.Set(\"composer-dll-directory\", os.path.abspath(''))\n", - "\n", - "# Set the data folder\n", - "Config.Set(\"data-folder\", '')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6ddc2ed2-5690-422c-8c91-6e6f64dd45cb", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# To generate the Security Identifier, we need to create and initialize the Map File Provider\n", - "# and call the SecurityIdentifier.GenerateEquity method\n", - "mapFileProvider = LocalZipMapFileProvider()\n", - "mapFileProvider.Initialize(DefaultDataProvider())\n", - "sid = SecurityIdentifier.GenerateEquity(\"SPY\", Market.USA, True, mapFileProvider, datetime(2022, 3, 1))\n", - "\n", - "qb = QuantBook()\n", - "symbol = Symbol(sid, \"SPY\")\n", - "history = qb.History(symbol, 3600, Resolution.Daily)\n", - "print(history)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/DataProcessing/process.sample.py b/DataProcessing/process.sample.py deleted file mode 100644 index 5b8285d..0000000 --- a/DataProcessing/process.sample.py +++ /dev/null @@ -1,32 +0,0 @@ -# CLRImports is required to handle Lean C# objects for Mapped Datasets (Single asset and Universe Selection) -# Requirements: -# python -m pip install clr-loader==0.1.7 -# python -m pip install pythonnet==3.0.0a2 -# This script must be executed in ./bin/Debug/net6.0 after the follwing command is executed -# dotnet build .\DataProcessing\ -import os -from CLRImports import * - -# To use QuantBook, we need to set its internal handlers -# We download LEAN confif with the default settings -with open("quantbook.json", 'w') as fp: - from requests import get - response = get("https://raw.githubusercontent.com/QuantConnect/Lean/master/Launcher/config.json") - fp.write(response.text) - -Config.SetConfigurationFile("quantbook.json") -Config.Set("composer-dll-directory", os.path.dirname(os.path.realpath(__file__))) - -# Set the data folder -Config.Set("data-folder", '') - -# To generate the Security Identifier, we need to create and initialize the Map File Provider -# and call the SecurityIdentifier.GenerateEquity method -mapFileProvider = LocalZipMapFileProvider() -mapFileProvider.Initialize(DefaultDataProvider()) -sid = SecurityIdentifier.GenerateEquity("SPY", Market.USA, True, mapFileProvider, datetime(2022, 3, 1)) - -qb = QuantBook() -symbol = Symbol(sid, "SPY") -history = qb.History(symbol, 3600, Resolution.Daily) -print(history) \ No newline at end of file diff --git a/DataProcessing/process.sample.sh b/DataProcessing/process.sample.sh deleted file mode 100644 index e69de29..0000000 diff --git a/DataProcessing/sync.sh b/DataProcessing/sync.sh new file mode 100644 index 0000000..e164442 --- /dev/null +++ b/DataProcessing/sync.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +processing_date="${QC_DATAFLEET_DEPLOYMENT_DATE}" +processing_date_yesterday="$(date -d "${processing_date} -1 days" +%Y%m%d)" +market="*" +include="*" +filter="_SPOT*" + +if [ -n "$2" ]; then + filter="${2}*" +fi + +if [ -n "$1" ]; then + market=$(echo "${1}" | tr [a-z] [A-Z]) + include="*${market}${filter}" + echo "Downloading crypto TAQ for market: ${market} include ${include}" +else + echo "Downloading crypto TAQ for all markets" +fi + +# We store AWS creds and set the CoinAPI creds instead +aws_access_key_id="${AWS_ACCESS_KEY_ID}" +aws_secret_access_key="${AWS_SECRET_ACCESS_KEY}" + +export AWS_ACCESS_KEY_ID="${COINAPI_AWS_ACCESS_KEY_ID}" +export AWS_SECRET_ACCESS_KEY="${COINAPI_AWS_SECRET_ACCESS_KEY}" + +# multipart corrupts files +aws configure set default.s3.multipart_threshold 100GB + +# We sync yesterdays file too because midnight data of 'processing_date' might be in the end of yesterdays file +aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/trades/$processing_date_yesterday/$market/ /raw/crypto/coinapi/trades/$processing_date_yesterday/ --exclude='*' --include="${include}" +aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/quotes/$processing_date_yesterday/$market/ /raw/crypto/coinapi/quotes/$processing_date_yesterday/ --exclude='*' --include="${include}" + +aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/trades/$processing_date/$market/ /raw/crypto/coinapi/trades/$processing_date/ --exclude='*' --include="${include}" +aws --endpoint-url=http://flatfiles.coinapi.io s3 sync s3://coinapi/quotes/$processing_date/$market/ /raw/crypto/coinapi/quotes/$processing_date/ --exclude='*' --include="${include}" + +# Restore AWS creds +export AWS_ACCESS_KEY_ID="$aws_access_key_id" +export AWS_SECRET_ACCESS_KEY="$aws_secret_access_key" \ No newline at end of file diff --git a/Demonstration.cs b/Demonstration.cs deleted file mode 100644 index fc0eb1f..0000000 --- a/Demonstration.cs +++ /dev/null @@ -1,77 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using QuantConnect.Data; -using QuantConnect.Util; -using QuantConnect.Orders; -using QuantConnect.Algorithm; -using QuantConnect.DataSource; - -namespace QuantConnect.DataLibrary.Tests -{ - /// - /// Example algorithm using the custom data type as a source of alpha - /// - public class CustomDataAlgorithm : QCAlgorithm - { - private Symbol _customDataSymbol; - private Symbol _equitySymbol; - - /// - /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. - /// - public override void Initialize() - { - SetStartDate(2013, 10, 07); //Set Start Date - SetEndDate(2013, 10, 11); //Set End Date - _equitySymbol = AddEquity("SPY").Symbol; - _customDataSymbol = AddData(_equitySymbol).Symbol; - } - - /// - /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. - /// - /// Slice object keyed by symbol containing the stock data - public override void OnData(Slice slice) - { - var data = slice.Get(); - if (!data.IsNullOrEmpty()) - { - // based on the custom data property we will buy or short the underlying equity - if (data[_customDataSymbol].SomeCustomProperty == "buy") - { - SetHoldings(_equitySymbol, 1); - } - else if (data[_customDataSymbol].SomeCustomProperty == "sell") - { - SetHoldings(_equitySymbol, -1); - } - } - } - - /// - /// Order fill event handler. On an order fill update the resulting information is passed to this method. - /// - /// Order event details containing details of the events - public override void OnOrderEvent(OrderEvent orderEvent) - { - if (orderEvent.Status.IsFill()) - { - Debug($"Purchased Stock: {orderEvent.Symbol}"); - } - } - } -} diff --git a/Demonstration.py b/Demonstration.py deleted file mode 100644 index 2f55e0d..0000000 --- a/Demonstration.py +++ /dev/null @@ -1,47 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from AlgorithmImports import * - -### -### Example algorithm using the custom data type as a source of alpha -### -class CustomDataAlgorithm(QCAlgorithm): - def Initialize(self): - ''' Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.''' - - self.SetStartDate(2020, 10, 7) #Set Start Date - self.SetEndDate(2020, 10, 11) #Set End Date - self.equity_symbol = self.AddEquity("SPY", Resolution.Daily).Symbol - self.custom_data_symbol = self.AddData(MyCustomDataType, self.equity_symbol).Symbol - - def OnData(self, slice): - ''' OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here. - - :param Slice slice: Slice object keyed by symbol containing the stock data - ''' - data = slice.Get(MyCustomDataType) - if data: - custom_data = data[self.custom_data_symbol] - if custom_data.SomeCustomProperty == "buy": - self.SetHoldings(self.equitySymbol, 1) - elif custom_data.SomeCustomProperty == "sell": - self.SetHoldings(self.equitySymbol, -1) - - def OnOrderEvent(self, orderEvent): - ''' Order fill event handler. On an order fill update the resulting information is passed to this method. - - :param OrderEvent orderEvent: Order event details containing details of the events - ''' - if orderEvent.Status == OrderStatus.Fill: - self.Debug(f'Purchased Stock: {orderEvent.Symbol}') \ No newline at end of file diff --git a/DemonstrationUniverse.cs b/DemonstrationUniverse.cs deleted file mode 100644 index d8b962c..0000000 --- a/DemonstrationUniverse.cs +++ /dev/null @@ -1,66 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using System.Linq; -using QuantConnect.Data; -using QuantConnect.Data.UniverseSelection; -using QuantConnect.DataSource; - -namespace QuantConnect.Algorithm.CSharp -{ - /// - /// Example algorithm using the custom data type as a source of alpha - /// - public class CustomDataUniverse : QCAlgorithm - { - /// - /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. - /// - public override void Initialize() - { - // Data ADDED via universe selection is added with Daily resolution. - UniverseSettings.Resolution = Resolution.Daily; - - SetStartDate(2022, 2, 14); - SetEndDate(2022, 2, 18); - SetCash(100000); - - // add a custom universe data source (defaults to usa-equity) - AddUniverse("MyCustomDataUniverseType", Resolution.Daily, data => - { - foreach (var datum in data) - { - Log($"{datum.Symbol},{datum.SomeCustomProperty},{datum.SomeNumericProperty}"); - } - - // define our selection criteria - return from d in data - where d.SomeCustomProperty == "buy" - select d.Symbol; - }); - } - - /// - /// Event fired each time that we add/remove securities from the data feed - /// - /// Security additions/removals for this time step - public override void OnSecuritiesChanged(SecurityChanges changes) - { - Log(changes.ToString()); - } - } -} \ No newline at end of file diff --git a/DemonstrationUniverse.py b/DemonstrationUniverse.py deleted file mode 100644 index 80b3657..0000000 --- a/DemonstrationUniverse.py +++ /dev/null @@ -1,50 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from AlgorithmImports import * - -### -### Example algorithm using the custom data type as a source of alpha -### -class CustomDataUniverse(QCAlgorithm): - def Initialize(self): - ''' Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized. ''' - - # Data ADDED via universe selection is added with Daily resolution. - self.UniverseSettings.Resolution = Resolution.Daily - - self.SetStartDate(2022, 2, 14) - self.SetEndDate(2022, 2, 18) - self.SetCash(100000) - - # add a custom universe data source (defaults to usa-equity) - self.AddUniverse(MyCustomDataUniverseType, "MyCustomDataUniverseType", Resolution.Daily, self.UniverseSelection) - - def UniverseSelection(self, data): - ''' Selected the securities - - :param List of MyCustomUniverseType data: List of MyCustomUniverseType - :return: List of Symbol objects ''' - - for datum in data: - self.Log(f"{datum.Symbol},{datum.Followers},{datum.DayPercentChange},{datum.WeekPercentChange}") - - # define our selection criteria - return [d.Symbol for d in data if d.SomeCustomProperty == 'buy'] - - def OnSecuritiesChanged(self, changes): - ''' Event fired each time that we add/remove securities from the data feed - - :param SecurityChanges changes: Security additions/removals for this time step - ''' - self.Log(changes.ToString()) \ No newline at end of file diff --git a/DropboxDownloader.py b/DropboxDownloader.py deleted file mode 100644 index 54b7e55..0000000 --- a/DropboxDownloader.py +++ /dev/null @@ -1,119 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to download files from a given dropbox directory. -# Files to be downloaded are filtered based on given date present in file name. - -# ARGUMENTS -# DROPBOX_API_KEY: Dropbox API KEY with read access. -# DROPBOX_SOURCE_DIRECTORY: path of the dropbox directory to search files within. -# DROPBOX_OUTPUT_DIRECTORY(optional): base path of the output directory to store to downloaded files. -# cmdline args expected in order: DROPBOX_API_KEY, DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE, DROPBOX_OUTPUT_DIRECTORY - -import requests -import json -import sys -import time -import os -from pathlib import Path - -DROPBOX_API_KEY = os.environ.get("DROPBOX_API_KEY") -DROPBOX_SOURCE_DIRECTORY = os.environ.get("DROPBOX_SOURCE_DIRECTORY") -QC_DATAFLEET_DEPLOYMENT_DATE = os.environ.get("QC_DATAFLEET_DEPLOYMENT_DATE") -DROPBOX_OUTPUT_DIRECTORY = os.environ.get("DROPBOX_OUTPUT_DIRECTORY", "/raw") - -def DownloadZipFile(filePath): - - print(f"Starting downloading file at: {filePath}") - - # defining the api-endpoint - API_ENDPOINT_DOWNLOAD = "https://content.dropboxapi.com/2/files/download" - - # data to be sent to api - data = {"path": filePath} - - headers = {"Authorization": f"Bearer {DROPBOX_API_KEY}", - "Dropbox-API-Arg": json.dumps(data)} - - # sending post request and saving response as response object - response = requests.post(url = API_ENDPOINT_DOWNLOAD, headers=headers) - - response.raise_for_status() # ensure we notice bad responses - - fileName = filePath.split("/")[-1] - outputPath = os.path.join(DROPBOX_OUTPUT_DIRECTORY, fileName) - - with open(outputPath, "wb") as f: - f.write(response.content) - print(f"Succesfully saved file at: {outputPath}") - -def GetFilePathsFromDate(targetLocation, dateString): - # defining the api-endpoint - API_ENDPOINT_FILEPATH = "https://api.dropboxapi.com/2/files/list_folder" - - headers = {"Content-Type": "application/json", - "Authorization": f"Bearer {DROPBOX_API_KEY}"} - - # data to be sent to api - data = {"path": targetLocation, - "recursive": False, - "include_media_info": False, - "include_deleted": False, - "include_has_explicit_shared_members": False, - "include_mounted_folders": True, - "include_non_downloadable_files": True} - - # sending post request and saving response as response object - response = requests.post(url = API_ENDPOINT_FILEPATH, headers=headers, data = json.dumps(data)) - - response.raise_for_status() # ensure we notice bad responses - - target_paths = [entry["path_display"] for entry in response.json()["entries"] if dateString in entry["path_display"]] - return target_paths - -def main(): - global DROPBOX_API_KEY, DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE, DROPBOX_OUTPUT_DIRECTORY - inputCount = len(sys.argv) - if inputCount > 1: - DROPBOX_API_KEY = sys.argv[1] - if inputCount > 2: - DROPBOX_SOURCE_DIRECTORY = sys.argv[2] - if inputCount > 3: - QC_DATAFLEET_DEPLOYMENT_DATE = sys.argv[3] - if inputCount > 4: - DROPBOX_OUTPUT_DIRECTORY = sys.argv[4] - - # make output path if doesn't exists - Path(DROPBOX_OUTPUT_DIRECTORY).mkdir(parents=True, exist_ok=True) - - target_paths = GetFilePathsFromDate(DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE) - print(f"Found {len(target_paths)} files with following paths {target_paths}") - - #download files - for path in target_paths: - count = 0 - maxTries = 3 - while True: - try: - DownloadZipFile(path) - break - except Exception as e: - count +=1 - if count > maxTries: - print(f"Error for file with path {path} --error message: {e}") - break - print(f"Error, sleep for 5 sec and retry download file with --path: {path}") - time.sleep(5) - -if __name__== "__main__": - main() diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Lean.DataSource.CoinAPI.sln b/Lean.DataSource.CoinAPI.sln new file mode 100644 index 0000000..1c6eb39 --- /dev/null +++ b/Lean.DataSource.CoinAPI.sln @@ -0,0 +1,37 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.002.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI", "QuantConnect.CoinAPI\QuantConnect.CoinAPI.csproj", "{2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI.Tests", "QuantConnect.CoinAPI.Tests\QuantConnect.CoinAPI.Tests.csproj", "{337CEE6E-639A-448D-95ED-2C1628E26AF2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataProcessing", "DataProcessing\DataProcessing.csproj", "{881514B4-641E-4EDC-8020-6BEA0CC8F48C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}.Release|Any CPU.Build.0 = Release|Any CPU + {337CEE6E-639A-448D-95ED-2C1628E26AF2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {337CEE6E-639A-448D-95ED-2C1628E26AF2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {337CEE6E-639A-448D-95ED-2C1628E26AF2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {337CEE6E-639A-448D-95ED-2C1628E26AF2}.Release|Any CPU.Build.0 = Release|Any CPU + {881514B4-641E-4EDC-8020-6BEA0CC8F48C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {881514B4-641E-4EDC-8020-6BEA0CC8F48C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {881514B4-641E-4EDC-8020-6BEA0CC8F48C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {881514B4-641E-4EDC-8020-6BEA0CC8F48C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {2C7F3105-1213-40AC-BEB7-3B5637C33F5D} + EndGlobalSection +EndGlobal diff --git a/MyCustomDataType.cs b/MyCustomDataType.cs deleted file mode 100644 index 8c8a025..0000000 --- a/MyCustomDataType.cs +++ /dev/null @@ -1,156 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NodaTime; -using ProtoBuf; -using System.IO; -using QuantConnect.Data; -using System.Collections.Generic; - -namespace QuantConnect.DataSource -{ - /// - /// Example custom data type - /// - [ProtoContract(SkipConstructor = true)] - public class MyCustomDataType : BaseData - { - /// - /// Some custom data property - /// - [ProtoMember(2000)] - public string SomeCustomProperty { get; set; } - - /// - /// Time passed between the date of the data and the time the data became available to us - /// - public TimeSpan Period { get; set; } = TimeSpan.FromDays(1); - - /// - /// Time the data became available - /// - public override DateTime EndTime => Time + Period; - - /// - /// Return the URL string source of the file. This will be converted to a stream - /// - /// Configuration object - /// Date of this source file - /// true if we're in live mode, false for backtesting mode - /// String URL of source file. - public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) - { - return new SubscriptionDataSource( - Path.Combine( - Globals.DataFolder, - "alternative", - "mycustomdatatype", - $"{config.Symbol.Value.ToLowerInvariant()}.csv" - ), - SubscriptionTransportMedium.LocalFile - ); - } - - /// - /// Parses the data from the line provided and loads it into LEAN - /// - /// Subscription configuration - /// Line of data - /// Date - /// Is live mode - /// New instance - public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) - { - var csv = line.Split(','); - - var parsedDate = Parse.DateTimeExact(csv[0], "yyyyMMdd"); - return new MyCustomDataType - { - Symbol = config.Symbol, - SomeCustomProperty = csv[1], - Time = parsedDate - Period, - }; - } - - /// - /// Clones the data - /// - /// A clone of the object - public override BaseData Clone() - { - return new MyCustomDataType - { - Symbol = Symbol, - Time = Time, - EndTime = EndTime, - SomeCustomProperty = SomeCustomProperty, - }; - } - - /// - /// Indicates whether the data source is tied to an underlying symbol and requires that corporate events be applied to it as well, such as renames and delistings - /// - /// false - public override bool RequiresMapping() - { - return true; - } - - /// - /// Indicates whether the data is sparse. - /// If true, we disable logging for missing files - /// - /// true - public override bool IsSparseData() - { - return true; - } - - /// - /// Converts the instance to string - /// - public override string ToString() - { - return $"{Symbol} - {SomeCustomProperty}"; - } - - /// - /// Gets the default resolution for this data and security type - /// - public override Resolution DefaultResolution() - { - return Resolution.Daily; - } - - /// - /// Gets the supported resolution for this data and security type - /// - public override List SupportedResolutions() - { - return DailyResolution; - } - - /// - /// Specifies the data time zone for this data type. This is useful for custom data types - /// - /// The of this data type - public override DateTimeZone DataTimeZone() - { - return DateTimeZone.Utc; - } - } -} diff --git a/MyCustomDataUniverseType.cs b/MyCustomDataUniverseType.cs deleted file mode 100644 index 5db2f4b..0000000 --- a/MyCustomDataUniverseType.cs +++ /dev/null @@ -1,141 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NodaTime; -using ProtoBuf; -using System.IO; -using QuantConnect.Data; -using System.Collections.Generic; -using System.Globalization; - -namespace QuantConnect.DataSource -{ - /// - /// Example custom data type - /// - [ProtoContract(SkipConstructor = true)] - public class MyCustomDataUniverseType : BaseData - { - /// - /// Some custom data property - /// - public string SomeCustomProperty { get; set; } - - /// - /// Some custom data property - /// - public decimal SomeNumericProperty { get; set; } - - /// - /// Time passed between the date of the data and the time the data became available to us - /// - public TimeSpan Period { get; set; } = TimeSpan.FromDays(1); - - /// - /// Time the data became available - /// - public override DateTime EndTime => Time + Period; - - /// - /// Return the URL string source of the file. This will be converted to a stream - /// - /// Configuration object - /// Date of this source file - /// true if we're in live mode, false for backtesting mode - /// String URL of source file. - public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) - { - return new SubscriptionDataSource( - Path.Combine( - Globals.DataFolder, - "alternative", - "mycustomdatatype", - "universe", - $"{date.ToStringInvariant(DateFormat.EightCharacter)}.csv" - ), - SubscriptionTransportMedium.LocalFile - ); - } - - /// - /// Parses the data from the line provided and loads it into LEAN - /// - /// Subscription configuration - /// Line of data - /// Date - /// Is live mode - /// New instance - public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode) - { - var csv = line.Split(','); - - var someNumericProperty = decimal.Parse(csv[2], NumberStyles.Any, CultureInfo.InvariantCulture); - - return new MyCustomDataUniverseType - { - Symbol = new Symbol(SecurityIdentifier.Parse(csv[0]), csv[1]), - SomeNumericProperty = someNumericProperty, - SomeCustomProperty = csv[3], - Time = date - Period, - Value = someNumericProperty - }; - } - - /// - /// Indicates whether the data is sparse. - /// If true, we disable logging for missing files - /// - /// true - public override bool IsSparseData() - { - return true; - } - - /// - /// Converts the instance to string - /// - public override string ToString() - { - return $"{Symbol} - {Value}"; - } - - /// - /// Gets the default resolution for this data and security type - /// - public override Resolution DefaultResolution() - { - return Resolution.Daily; - } - - /// - /// Gets the supported resolution for this data and security type - /// - public override List SupportedResolutions() - { - return DailyResolution; - } - - /// - /// Specifies the data time zone for this data type. This is useful for custom data types - /// - /// The of this data type - public override DateTimeZone DataTimeZone() - { - return DateTimeZone.Utc; - } - } -} \ No newline at end of file diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs new file mode 100644 index 0000000..669d48a --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs @@ -0,0 +1,110 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Util; +using QuantConnect.Logging; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinAPIDataDownloaderTests + { + private CoinAPIDataDownloader _downloader; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _downloader = new(); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + _downloader.DisposeSafely(); + } + + private static IEnumerable HistoricalValidDataTestCases + { + get + { + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Minute, new DateTime(2024, 1, 1, 20, 0, 0), new DateTime(2024, 1, 1, 21, 0, 0)); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDKraken, Resolution.Minute, new DateTime(2024, 1, 1, 20, 0, 0), new DateTime(2024, 1, 1, 21, 0, 0)); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDBitfinex, Resolution.Hour, new DateTime(2024, 1, 1, 0, 0, 0), new DateTime(2024, 1, 1, 12, 0, 0)); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Daily, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1)); + } + } + + [TestCaseSource(nameof(HistoricalValidDataTestCases))] + public void DownloadsHistoricalDataWithValidDataTestParameters(Symbol symbol, Resolution resolution, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + { + var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, TickType.Trade); + + var downloadResponse = _downloader.Get(parameters).ToList(); + + Assert.IsNotEmpty(downloadResponse); + + Log.Trace($"{symbol}.{resolution}.[{startDateTimeUtc} - {endDateTimeUtc}]: Amount = {downloadResponse.Count}"); + + CoinApiTestHelper.AssertSymbol(downloadResponse.First().Symbol, symbol); + + CoinApiTestHelper.AssertBaseData(downloadResponse, resolution); + } + + private static IEnumerable HistoricalInvalidDataTestCases + { + get + { + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Tick, new DateTime(2024, 1, 1, 20, 0, 0), new DateTime(2024, 1, 1, 21, 0, 0), TickType.Trade) + .SetDescription($"Not supported - {Resolution.Tick}"); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Tick, new DateTime(2024, 1, 1), new DateTime(2023, 1, 1), TickType.Trade) + .SetDescription("Wrong startDateTime - startDateTime > endDateTime"); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Tick, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.Quote) + .SetDescription($"Not supported - {TickType.Quote}"); + yield return new TestCaseData(CoinApiTestHelper.BTCUSDTBinance, Resolution.Tick, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.OpenInterest) + .SetDescription($"Not supported - {TickType.OpenInterest}"); + } + } + + [TestCaseSource(nameof(HistoricalInvalidDataTestCases))] + public void DownloadsHistoricalDataWithInvalidDataTestParameters(Symbol symbol, Resolution resolution, DateTime startDateTimeUtc, DateTime endDateTimeUtc, TickType tickType) + { + var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, tickType); + + var downloadResponse = _downloader.Get(parameters).ToList(); + + Assert.IsEmpty(downloadResponse); + } + + private static IEnumerable HistoricalInvalidDataThrowExceptionTestCases + { + get + { + yield return new TestCaseData(Symbol.Create("BTCBTC", SecurityType.Crypto, Market.Binance)) + .SetDescription($"Wrong Symbol - 'BTCBTC'"); + yield return new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Equity, Market.Binance)) + .SetDescription($"Wrong SecurityType - {SecurityType.Equity}"); + } + } + + [TestCaseSource(nameof(HistoricalInvalidDataThrowExceptionTestCases))] + public void DownloadsHistoricalDataWithInvalidDataTestParametersThrowException(Symbol symbol) + { + var parameters = new DataDownloaderGetParameters(symbol, Resolution.Minute, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.Trade); + + Assert.That(() => _downloader.Get(parameters).ToList(), Throws.Exception); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs new file mode 100644 index 0000000..a7c93fd --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs @@ -0,0 +1,126 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Data; +using QuantConnect.Util; +using QuantConnect.Data.Market; +using QuantConnect.Securities; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinAPIHistoryProviderTests + { + private static readonly Symbol _CoinbaseBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase); + private static readonly Symbol _BitfinexBtcUsdSymbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Bitfinex); + private CoinApiDataQueueHandlerMock _coinApiDataQueueHandler; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _coinApiDataQueueHandler = new CoinApiDataQueueHandlerMock(); + } + + // -- DATA TO TEST -- + private static TestCaseData[] TestData => new[] + { + // No data - invalid resolution or data type, or period is more than limit + new TestCaseData(_BitfinexBtcUsdSymbol, Resolution.Tick, typeof(TradeBar), 100, false), + new TestCaseData(_BitfinexBtcUsdSymbol, Resolution.Daily, typeof(QuoteBar), 100, false), + // Has data + new TestCaseData(_BitfinexBtcUsdSymbol, Resolution.Minute, typeof(TradeBar), 216, true), + new TestCaseData(_CoinbaseBtcUsdSymbol, Resolution.Minute, typeof(TradeBar), 342, true), + new TestCaseData(_CoinbaseBtcUsdSymbol, Resolution.Hour, typeof(TradeBar), 107, true), + new TestCaseData(_CoinbaseBtcUsdSymbol, Resolution.Daily, typeof(TradeBar), 489, true), + // Can get data for resolution second + new TestCaseData(_BitfinexBtcUsdSymbol, Resolution.Second, typeof(TradeBar), 300, true) + }; + + [Test] + [TestCaseSource(nameof(TestData))] + public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNonEmptyResult) + { + _coinApiDataQueueHandler.SetUpHistDataLimit(100); + + var nowUtc = DateTime.UtcNow; + var periodTimeSpan = TimeSpan.FromTicks(resolution.ToTimeSpan().Ticks * period); + var startTimeUtc = nowUtc.Add(-periodTimeSpan); + + var historyRequests = new[] + { + new HistoryRequest(startTimeUtc, nowUtc, dataType, symbol, resolution, + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), TimeZones.Utc, + resolution, true, false, DataNormalizationMode.Raw, TickType.Trade) + }; + + var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc).ToArray(); + + if (isNonEmptyResult) + { + // For resolution larger than second do more tests + if (resolution > Resolution.Second) + { + Assert.AreEqual(period, slices.Length); + + var firstSliceTradeBars = slices.First().Bars.Values; + + Assert.True(firstSliceTradeBars.Select(x => x.Symbol).Contains(symbol)); + + firstSliceTradeBars.DoForEach(tb => + { + var resTimeSpan = resolution.ToTimeSpan(); + Assert.AreEqual(resTimeSpan, tb.Period); + Assert.AreEqual(startTimeUtc.RoundUp(resTimeSpan), tb.Time); + }); + + var lastSliceTradeBars = slices.Last().Bars.Values; + + lastSliceTradeBars.DoForEach(tb => + { + var resTimeSpan = resolution.ToTimeSpan(); + Assert.AreEqual(resTimeSpan, tb.Period); + Assert.AreEqual(nowUtc.RoundDown(resTimeSpan), tb.Time); + }); + } + // For res. second data counts, start/end dates may slightly vary from historical request's + // Make sure just that resolution is correct and amount is positive numb. + else + { + Assert.IsTrue(slices.Length > 0); + Assert.AreEqual(resolution.ToTimeSpan(), slices.First().Bars.Values.FirstOrDefault()?.Period); + } + + // Slices are ordered by time + Assert.That(slices, Is.Ordered.By("Time")); + } + else + { + // Empty + Assert.IsEmpty(slices); + } + } + + public class CoinApiDataQueueHandlerMock : CoinApiDataQueueHandler + { + public new void SetUpHistDataLimit(int limit) + { + base.SetUpHistDataLimit(limit); + } + } + + } +} diff --git a/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs new file mode 100644 index 0000000..7303fc1 --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs @@ -0,0 +1,79 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinAPISymbolMapperTests + { + private CoinApiSymbolMapper _coinApiSymbolMapper; + + [OneTimeSetUp] + public void SetUp() + { + _coinApiSymbolMapper = new CoinApiSymbolMapper(); + } + + [TestCase("COINBASE_SPOT_BTC_USD", "BTCUSD", Market.Coinbase)] + [TestCase("COINBASE_SPOT_BCH_USD", "BCHUSD", Market.Coinbase)] + [TestCase("BITFINEX_SPOT_BTC_USD", "BTCUSD", Market.Bitfinex)] + [TestCase("BITFINEX_SPOT_BCHABC_USD", "BCHUSD", Market.Bitfinex)] + [TestCase("BITFINEX_SPOT_BCHSV_USD", "BSVUSD", Market.Bitfinex)] + [TestCase("BITFINEX_SPOT_ABS_USD", "ABYSSUSD", Market.Bitfinex)] + public void ReturnsCorrectLeanSymbol(string coinApiSymbolId, string leanTicker, string market) + { + var symbol = _coinApiSymbolMapper.GetLeanSymbol(coinApiSymbolId, SecurityType.Crypto, string.Empty); + + Assert.That(symbol.Value, Is.EqualTo(leanTicker)); + Assert.That(symbol.ID.SecurityType, Is.EqualTo(SecurityType.Crypto)); + Assert.That(symbol.ID.Market, Is.EqualTo(market)); + } + + [TestCase("BTCUSD", Market.Coinbase, "COINBASE_SPOT_BTC_USD")] + [TestCase("BCHUSD", Market.Coinbase, "COINBASE_SPOT_BCH_USD")] + [TestCase("BTCUSD", Market.Bitfinex, "BITFINEX_SPOT_BTC_USD")] + [TestCase("BCHUSD", Market.Bitfinex, "BITFINEX_SPOT_BCHABC_USD")] + [TestCase("BSVUSD", Market.Bitfinex, "BITFINEX_SPOT_BCHSV_USD")] + [TestCase("ABYSSUSD", Market.Bitfinex, "BITFINEX_SPOT_ABS_USD")] + public void ReturnsCorrectBrokerageSymbol(string leanTicker, string market, string coinApiSymbolId) + { + var symbol = Symbol.Create(leanTicker, SecurityType.Crypto, market); + + var symbolId = _coinApiSymbolMapper.GetBrokerageSymbol(symbol); + + Assert.That(symbolId, Is.EqualTo(coinApiSymbolId)); + } + + [TestCase("BTCUSDT", Market.Binance, "BINANCEFTS_PERP_BTC_USDT")] + public void ReturnsCorrectBrokerageFutureSymbol(string leanTicker, string market, string coinApiSymbolId) + { + var symbol = Symbol.Create(leanTicker, SecurityType.CryptoFuture, market); + + var symbolId = _coinApiSymbolMapper.GetBrokerageSymbol(symbol); + + Assert.That(symbolId, Is.EqualTo(coinApiSymbolId)); + } + + [TestCase("BTCUSDT", Market.Kraken)] + public void TryGetWrongBrokerageFutureSymbolThrowException(string leanTicker, string market) + { + var symbol = Symbol.Create(leanTicker, SecurityType.CryptoFuture, market); + Assert.Throws(() => _coinApiSymbolMapper.GetBrokerageSymbol(symbol)); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs new file mode 100644 index 0000000..97ee813 --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs @@ -0,0 +1,38 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Configuration; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinApiAdditionalTests + { + [Test] + public void ThrowsOnFailedAuthentication() + { + Config.Set("coinapi-api-key", "wrong-api-key"); + + Assert.Throws(() => + { + using var _coinApiDataQueueHandler = new CoinApiDataQueueHandler(); + }); + + // reset api key + TestSetup.GlobalSetup(); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs new file mode 100644 index 0000000..c63d45a --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs @@ -0,0 +1,235 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Data; +using QuantConnect.Util; +using QuantConnect.Logging; +using QuantConnect.Data.Market; +using System.Collections.Concurrent; + +namespace QuantConnect.CoinAPI.Tests +{ + [TestFixture] + public class CoinApiDataQueueHandlerTest + { + private CoinApiDataQueueHandler _coinApiDataQueueHandler; + private CancellationTokenSource _cancellationTokenSource; + + [SetUp] + public void SetUp() + { + _coinApiDataQueueHandler = new(); + _cancellationTokenSource = new(); + } + + [TearDown] + public void TearDown() + { + _cancellationTokenSource.Dispose(); + + if (_coinApiDataQueueHandler != null) + { + _coinApiDataQueueHandler.Dispose(); + } + } + + [Test] + public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest() + { + var resetEvent = new AutoResetEvent(false); + var tradeBars = new List(); + var resolution = Resolution.Second; + var symbol = CoinApiTestHelper.BTCUSDCoinbase; + var dataConfig = CoinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution); + + ProcessFeed( + _coinApiDataQueueHandler.Subscribe(dataConfig, (s, e) => { }), + _cancellationTokenSource.Token, + tick => + { + Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest)}: {tick}"); + tradeBars.Add(tick); + + if (tradeBars.Count > 5) + { + resetEvent.Set(); + } + }, + () => _cancellationTokenSource.Cancel()); + + Assert.IsTrue(resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token)); + + _coinApiDataQueueHandler.Unsubscribe(dataConfig); + + CoinApiTestHelper.AssertSymbol(tradeBars.First().Symbol, symbol); + + CoinApiTestHelper.AssertBaseData(tradeBars, resolution); + + _cancellationTokenSource.Cancel(); + } + + [Test] + public void SubscribeToBTCUSDSecondOnDifferentMarkets() + { + var resetEvent = new AutoResetEvent(false); + var tradeBars = new List(); + var resolution = Resolution.Second; + var minimDataFromExchange = 5; + + var symbolBaseData = new ConcurrentDictionary> + { + [CoinApiTestHelper.BTCUSDKraken] = new(), + [CoinApiTestHelper.BTCUSDTBinance] = new(), + [CoinApiTestHelper.BTCUSDBitfinex] = new(), + [CoinApiTestHelper.BTCUSDCoinbase] = new() + }; + + var dataConfigs = new List(); + foreach (var symbol in symbolBaseData.Keys) + { + dataConfigs.Add(CoinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution)); + } + + foreach (var config in dataConfigs) + { + ProcessFeed( + _coinApiDataQueueHandler.Subscribe(config, (s, e) => { }), + _cancellationTokenSource.Token, + tick => + { + Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnDifferentMarkets)}: {tick}"); + symbolBaseData[tick.Symbol].Add(tick); + }, + () => + { + _cancellationTokenSource.Cancel(); + }); + } + + resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token); + + foreach (var data in symbolBaseData) + { + if (data.Value.Count > minimDataFromExchange) + { + Log.Debug($"Unsubscribe: Symbol: {data.Key}, BaseData.Count: {data.Value.Count}"); + var config = dataConfigs.Where(x => x.Symbol == data.Key).First(); + _coinApiDataQueueHandler.Unsubscribe(config); + dataConfigs.Remove(config); + } + } + + if (dataConfigs.Count != 0) + { + resetEvent.WaitOne(TimeSpan.FromSeconds(30), _cancellationTokenSource.Token); + } + + foreach (var config in dataConfigs) + { + _coinApiDataQueueHandler.Unsubscribe(config); + } + + foreach (var data in symbolBaseData.Values) + { + CoinApiTestHelper.AssertBaseData(data, resolution); + } + + _cancellationTokenSource.Cancel(); + } + + [Test] + public void SubscribeToBTCUSDTFutureSecondBinance() + { + var resetEvent = new AutoResetEvent(false); + var resolution = Resolution.Second; + var tickData = new List(); + var symbol = CoinApiTestHelper.BTCUSDFutureBinance; + var config = CoinApiTestHelper.GetSubscriptionDataConfigs(symbol, resolution); + + ProcessFeed( + _coinApiDataQueueHandler.Subscribe(config, (s, e) => { }), + _cancellationTokenSource.Token, + tick => + { + Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)}: {tick}"); + tickData.Add(tick); + + if (tickData.Count > 5) + { + resetEvent.Set(); + } + }, + () => + { + _cancellationTokenSource.Cancel(); + }); + + resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token); + + // if seq is empty, give additional chance + if (tickData.Count == 0) + { + resetEvent.WaitOne(TimeSpan.FromSeconds(60), _cancellationTokenSource.Token); + } + + _coinApiDataQueueHandler.Unsubscribe(config); + + if (tickData.Count == 0) + { + Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}"); + } + + CoinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol); + + CoinApiTestHelper.AssertBaseData(tickData, resolution); + + _cancellationTokenSource.Cancel(); + } + + private Task ProcessFeed(IEnumerator enumerator, CancellationToken cancellationToken, Action? callback = null, Action? throwExceptionCallback = null) + { + return Task.Factory.StartNew(() => + { + try + { + while (enumerator.MoveNext() && !cancellationToken.IsCancellationRequested) + { + BaseData tick = enumerator.Current; + + if (tick != null) + { + callback?.Invoke(tick); + } + + cancellationToken.WaitHandle.WaitOne(TimeSpan.FromMilliseconds(500)); + } + } + catch + { + throw; + } + }, cancellationToken).ContinueWith(task => + { + if (throwExceptionCallback != null) + { + throwExceptionCallback(); + } + Log.Error("The throwExceptionCallback is null."); + }, TaskContinuationOptions.OnlyOnFaulted); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs new file mode 100644 index 0000000..849a1c8 --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs @@ -0,0 +1,100 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Data; +using QuantConnect.Logging; +using QuantConnect.Data.Market; + +namespace QuantConnect.CoinAPI.Tests +{ + public static class CoinApiTestHelper + { + public static readonly Symbol BTCUSDKraken = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Kraken); + public static readonly Symbol BTCUSDBitfinex = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Bitfinex); + public static readonly Symbol BTCUSDCoinbase = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.Coinbase); + public static readonly Symbol BTCUSDTBinance = Symbol.Create("BTCUSDT", SecurityType.Crypto, Market.Binance); + public static readonly Symbol BTCUSDTBinanceUS = Symbol.Create("BTCUSDT", SecurityType.Crypto, Market.BinanceUS); + + /// + /// PERPETUAL BTCUSDT + /// + public static readonly Symbol BTCUSDFutureBinance = Symbol.Create("BTCUSD", SecurityType.CryptoFuture, Market.Binance); + + public static void AssertSymbol(Symbol actualSymbol, Symbol expectedSymbol) + { + Assert.IsTrue(actualSymbol == expectedSymbol, $"Unexpected Symbol: Expected {expectedSymbol}, but received {actualSymbol}."); + } + + public static void AssertBaseData(List tradeBars, Resolution expectedResolution) + { + Assert.Greater(tradeBars.Count, 0); + foreach (var tick in tradeBars) + { + Assert.IsNotNull(tick); + Assert.Greater(tick.Price, 0); + Assert.Greater(tick.Value, 0); + Assert.Greater(tick.Time, DateTime.UnixEpoch); + Assert.Greater(tick.EndTime, DateTime.UnixEpoch); + + if (tick.DataType == MarketDataType.Tick) + { + return; + } + + Assert.IsTrue(tick.DataType == MarketDataType.TradeBar || tick.DataType == MarketDataType.QuoteBar, $"Unexpected data type: Expected TradeBar or QuoteBar, but received {tick.DataType}."); + + switch (tick) + { + case TradeBar trade: + Assert.Greater(trade.Low, 0); + Assert.Greater(trade.Open, 0); + Assert.Greater(trade.High, 0); + Assert.Greater(trade.Close, 0); + Assert.Greater(trade.Volume, 0); + Assert.IsTrue(trade.Period.ToHigherResolutionEquivalent(true) == expectedResolution); + break; + default: + Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(AssertBaseData)}: The tick type doesn't support"); + break; + } + } + } + + public static SubscriptionDataConfig GetSubscriptionDataConfigs(Symbol symbol, Resolution resolution) + { + return GetSubscriptionDataConfig(symbol, resolution); + } + + public static SubscriptionDataConfig GetSubscriptionTickDataConfigs(Symbol symbol) + { + return new SubscriptionDataConfig(GetSubscriptionDataConfig(symbol, Resolution.Tick), tickType: TickType.Trade); + } + + private static SubscriptionDataConfig GetSubscriptionDataConfig(Symbol symbol, Resolution resolution) + { + return new SubscriptionDataConfig( + typeof(T), + symbol, + resolution, + TimeZones.Utc, + TimeZones.Utc, + true, + extendedHours: false, + false); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj b/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj new file mode 100644 index 0000000..d73485b --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj @@ -0,0 +1,42 @@ + + + + Release + AnyCPU + net6.0 + bin\$(Configuration)\ + QuantConnect.CoinAPI.Tests + QuantConnect.CoinAPI.Tests + QuantConnect.CoinAPI.Tests + QuantConnect.CoinAPI.Tests + false + enable + enable + + false + true + UnitTest + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + PreserveNewest + + + + diff --git a/QuantConnect.CoinAPI.Tests/TestSetup.cs b/QuantConnect.CoinAPI.Tests/TestSetup.cs new file mode 100644 index 0000000..05bf2fb --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/TestSetup.cs @@ -0,0 +1,64 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using System.Collections; +using QuantConnect.Logging; +using QuantConnect.Configuration; + +namespace QuantConnect.CoinAPI.Tests +{ + [SetUpFixture] + public static class TestSetup + { + [OneTimeSetUp] + public static void GlobalSetup() + { + // Log.DebuggingEnabled = true; + Log.LogHandler = new CompositeLogHandler(); + Log.Trace("TestSetup(): starting..."); + ReloadConfiguration(); + } + + private static void ReloadConfiguration() + { + // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder + var dir = TestContext.CurrentContext.TestDirectory; + Environment.CurrentDirectory = dir; + Directory.SetCurrentDirectory(dir); + // reload config from current path + Config.Reset(); + + var environment = Environment.GetEnvironmentVariables(); + foreach (DictionaryEntry entry in environment) + { + var envKey = entry.Key.ToString(); + var value = entry.Value.ToString(); + + if (envKey.StartsWith("QC_")) + { + var key = envKey.Substring(3).Replace("_", "-").ToLower(); + + Log.Trace($"TestSetup(): Updating config setting '{key}' from environment var '{envKey}'"); + Config.Set(key, value); + } + } + + // resets the version among other things + Globals.Reset(); + } + } +} diff --git a/QuantConnect.CoinAPI.Tests/config.json b/QuantConnect.CoinAPI.Tests/config.json new file mode 100644 index 0000000..6241a29 --- /dev/null +++ b/QuantConnect.CoinAPI.Tests/config.json @@ -0,0 +1,7 @@ +{ + "data-folder": "../../../../Lean/Data/", + "data-directory": "../../../../Lean/Data/", + + "coinapi-api-key": "", + "coinapi-product": "free" +} diff --git a/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs new file mode 100644 index 0000000..5f9e336 --- /dev/null +++ b/QuantConnect.CoinAPI/CoinAPIDataDownloader.cs @@ -0,0 +1,78 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Data; +using QuantConnect.Util; +using QuantConnect.Logging; +using QuantConnect.Securities; +using QuantConnect.Data.Market; + +namespace QuantConnect.CoinAPI +{ + public class CoinAPIDataDownloader : IDataDownloader, IDisposable + { + private readonly CoinApiDataQueueHandler _historyProvider; + + private readonly MarketHoursDatabase _marketHoursDatabase; + + public CoinAPIDataDownloader() + { + _historyProvider = new CoinApiDataQueueHandler(); + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + } + + public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetParameters) + { + if (dataDownloaderGetParameters.TickType != TickType.Trade) + { + Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}: Not supported data type - {dataDownloaderGetParameters.TickType}. " + + $"Currently available support only for historical of type - {nameof(TickType.Trade)}"); + yield break; + } + + if (dataDownloaderGetParameters.EndUtc < dataDownloaderGetParameters.StartUtc) + { + Log.Error($"{nameof(CoinAPIDataDownloader)}.{nameof(Get)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); + yield break; + } + + var symbol = dataDownloaderGetParameters.Symbol; + + var historyRequests = new HistoryRequest( + startTimeUtc: dataDownloaderGetParameters.StartUtc, + endTimeUtc: dataDownloaderGetParameters.EndUtc, + dataType: typeof(TradeBar), + symbol: symbol, + resolution: dataDownloaderGetParameters.Resolution, + exchangeHours: _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType), + dataTimeZone: _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType), + fillForwardResolution: dataDownloaderGetParameters.Resolution, + includeExtendedMarketHours: true, + isCustomData: false, + dataNormalizationMode: DataNormalizationMode.Raw, + tickType: TickType.Trade); + + foreach (var slice in _historyProvider.GetHistory(historyRequests)) + { + yield return slice; + } + } + + public void Dispose() + { + _historyProvider.DisposeSafely(); + } + } +} diff --git a/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs new file mode 100644 index 0000000..6f067ff --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApi.HistoryProvider.cs @@ -0,0 +1,169 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NodaTime; +using RestSharp; +using Newtonsoft.Json; +using QuantConnect.Data; +using QuantConnect.Logging; +using QuantConnect.Data.Market; +using QuantConnect.CoinAPI.Messages; +using QuantConnect.Lean.Engine.DataFeeds; +using HistoryRequest = QuantConnect.Data.HistoryRequest; +using QuantConnect.CoinAPI.Models; + +namespace QuantConnect.CoinAPI +{ + public partial class CoinApiDataQueueHandler + { + private readonly RestClient restClient = new RestClient(); + + private readonly RestRequest restRequest = new(Method.GET); + + /// + /// Indicates whether the warning for invalid history has been fired. + /// + private bool _invalidHistoryDataTypeWarningFired; + + public override void Initialize(HistoryProviderInitializeParameters parameters) + { + // NOP + } + + public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + { + var subscriptions = new List(); + foreach (var request in requests) + { + var history = GetHistory(request); + var subscription = CreateSubscription(request, history); + subscriptions.Add(subscription); + } + return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); + } + + public IEnumerable GetHistory(HistoryRequest historyRequest) + { + if (historyRequest.Symbol.SecurityType != SecurityType.Crypto && historyRequest.Symbol.SecurityType != SecurityType.CryptoFuture) + { + Log.Error($"CoinApiDataQueueHandler.GetHistory(): Invalid security type {historyRequest.Symbol.SecurityType}"); + yield break; + } + + if (historyRequest.Resolution == Resolution.Tick) + { + Log.Error($"CoinApiDataQueueHandler.GetHistory(): No historical ticks, only OHLCV timeseries"); + yield break; + } + + if (historyRequest.DataType == typeof(QuoteBar)) + { + if (!_invalidHistoryDataTypeWarningFired) + { + Log.Error("CoinApiDataQueueHandler.GetHistory(): No historical QuoteBars , only TradeBars"); + _invalidHistoryDataTypeWarningFired = true; + } + yield break; + } + + var resolutionTimeSpan = historyRequest.Resolution.ToTimeSpan(); + var lastRequestedBarStartTime = historyRequest.EndTimeUtc.RoundDown(resolutionTimeSpan); + var currentStartTime = historyRequest.StartTimeUtc.RoundUp(resolutionTimeSpan); + var currentEndTime = lastRequestedBarStartTime; + + // Perform a check of the number of bars requested, this must not exceed a static limit + var dataRequestedCount = (currentEndTime - currentStartTime).Ticks + / resolutionTimeSpan.Ticks; + + if (dataRequestedCount > HistoricalDataPerRequestLimit) + { + currentEndTime = currentStartTime + + TimeSpan.FromTicks(resolutionTimeSpan.Ticks * HistoricalDataPerRequestLimit); + } + + while (currentStartTime < lastRequestedBarStartTime) + { + var coinApiSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); + var coinApiPeriod = _ResolutionToCoinApiPeriodMappings[historyRequest.Resolution]; + + // Time must be in ISO 8601 format + var coinApiStartTime = currentStartTime.ToStringInvariant("s"); + var coinApiEndTime = currentEndTime.ToStringInvariant("s"); + + // Construct URL for rest request + restClient.BaseUrl = new Uri("https://rest.coinapi.io/v1/ohlcv/" + + $"{coinApiSymbol}/history?period_id={coinApiPeriod}&limit={HistoricalDataPerRequestLimit}" + + $"&time_start={coinApiStartTime}&time_end={coinApiEndTime}"); + + restRequest.AddOrUpdateHeader("X-CoinAPI-Key", _apiKey); + var response = restClient.Execute(restRequest); + + // Log the information associated with the API Key's rest call limits. + TraceRestUsage(response); + + HistoricalDataMessage[]? coinApiHistoryBars; + try + { + // Deserialize to array + coinApiHistoryBars = JsonConvert.DeserializeObject(response.Content); + } + catch (JsonSerializationException) + { + var error = JsonConvert.DeserializeObject(response.Content); + throw new Exception(error.Error); + } + + // Can be no historical data for a short period interval + if (!coinApiHistoryBars.Any()) + { + Log.Error($"CoinApiDataQueueHandler.GetHistory(): API returned no data for the requested period [{coinApiStartTime} - {coinApiEndTime}] for symbol [{historyRequest.Symbol}]"); + continue; + } + + foreach (var ohlcv in coinApiHistoryBars) + { + yield return + new TradeBar(ohlcv.TimePeriodStart, historyRequest.Symbol, ohlcv.PriceOpen, ohlcv.PriceHigh, + ohlcv.PriceLow, ohlcv.PriceClose, ohlcv.VolumeTraded, historyRequest.Resolution.ToTimeSpan()); + } + + currentStartTime = currentEndTime; + currentEndTime += TimeSpan.FromTicks(resolutionTimeSpan.Ticks * HistoricalDataPerRequestLimit); + } + } + + private void TraceRestUsage(IRestResponse response) + { + var total = GetHttpHeaderValue(response, "x-ratelimit-limit"); + var used = GetHttpHeaderValue(response, "x-ratelimit-used"); + var remaining = GetHttpHeaderValue(response, "x-ratelimit-remaining"); + + Log.Trace($"CoinApiDataQueueHandler.TraceRestUsage(): Used {used}, Remaining {remaining}, Total {total}"); + } + + private string GetHttpHeaderValue(IRestResponse response, string propertyName) + { + return response.Headers + .FirstOrDefault(x => x.Name == propertyName)? + .Value.ToString(); + } + + // WARNING: here to be called from tests to reduce explicitly the amount of request's output + protected void SetUpHistDataLimit(int limit) + { + HistoricalDataPerRequestLimit = limit; + } + } +} diff --git a/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs new file mode 100644 index 0000000..a2e612b --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApiDataQueueHandler.cs @@ -0,0 +1,526 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using RestSharp; +using System.Net; +using System.Text; +using Newtonsoft.Json; +using QuantConnect.Api; +using QuantConnect.Data; +using QuantConnect.Util; +using QuantConnect.Packets; +using QuantConnect.Logging; +using Newtonsoft.Json.Linq; +using CoinAPI.WebSocket.V1; +using QuantConnect.Interfaces; +using QuantConnect.Data.Market; +using QuantConnect.Configuration; +using System.Security.Cryptography; +using System.Net.NetworkInformation; +using System.Collections.Concurrent; +using CoinAPI.WebSocket.V1.DataModels; +using QuantConnect.Lean.Engine.HistoricalData; + +namespace QuantConnect.CoinAPI +{ + /// + /// An implementation of for CoinAPI + /// + public partial class CoinApiDataQueueHandler : SynchronizingHistoryProvider, IDataQueueHandler + { + protected int HistoricalDataPerRequestLimit = 10000; + private static readonly Dictionary _ResolutionToCoinApiPeriodMappings = new Dictionary + { + { Resolution.Second, "1SEC"}, + { Resolution.Minute, "1MIN" }, + { Resolution.Hour, "1HRS" }, + { Resolution.Daily, "1DAY" }, + }; + + private readonly string _apiKey = Config.Get("coinapi-api-key"); + private readonly string[] _streamingDataType; + private readonly CoinApiWsClient _client; + private readonly object _locker = new object(); + private ConcurrentDictionary _symbolCache = new ConcurrentDictionary(); + private readonly CoinApiSymbolMapper _symbolMapper = new CoinApiSymbolMapper(); + private readonly IDataAggregator _dataAggregator; + private readonly EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; + + private readonly TimeSpan _subscribeDelay = TimeSpan.FromMilliseconds(250); + private readonly object _lockerSubscriptions = new object(); + private DateTime _lastSubscribeRequestUtcTime = DateTime.MinValue; + private bool _subscriptionsPending; + + private readonly TimeSpan _minimumTimeBetweenHelloMessages = TimeSpan.FromSeconds(5); + private DateTime _nextHelloMessageUtcTime = DateTime.MinValue; + + private readonly ConcurrentDictionary _previousQuotes = new ConcurrentDictionary(); + + /// + /// Initializes a new instance of the class + /// + public CoinApiDataQueueHandler() + { + _dataAggregator = Composer.Instance.GetPart(); + if (_dataAggregator == null) + { + _dataAggregator = + Composer.Instance.GetExportedValueByTypeName(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); + } + var product = Config.GetValue("coinapi-product"); + _streamingDataType = product < CoinApiProduct.Streamer + ? new[] { "trade" } + : new[] { "trade", "quote" }; + + Log.Trace($"{nameof(CoinApiDataQueueHandler)}: using plan '{product}'. Available data types: '{string.Join(",", _streamingDataType)}'"); + + ValidateSubscription(); + + _client = new CoinApiWsClient(); + _client.TradeEvent += OnTrade; + _client.QuoteEvent += OnQuote; + _client.Error += OnError; + _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager(); + _subscriptionManager.SubscribeImpl += (s, t) => Subscribe(s); + _subscriptionManager.UnsubscribeImpl += (s, t) => Unsubscribe(s); + } + + /// + /// Subscribe to the specified configuration + /// + /// defines the parameters to subscribe to a data feed + /// handler to be fired on new data available + /// The new enumerator for this subscription request + public IEnumerator Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) + { + if (!CanSubscribe(dataConfig.Symbol)) + { + return null; + } + + var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); + _subscriptionManager.Subscribe(dataConfig); + + return enumerator; + } + + /// + /// Sets the job we're subscribing for + /// + /// Job we're subscribing for + public void SetJob(LiveNodePacket job) + { + } + + /// + /// Adds the specified symbols to the subscription + /// + /// The symbols to be added keyed by SecurityType + private bool Subscribe(IEnumerable symbols) + { + ProcessSubscriptionRequest(); + return true; + } + + /// + /// Removes the specified configuration + /// + /// Subscription config to be removed + public void Unsubscribe(SubscriptionDataConfig dataConfig) + { + _subscriptionManager.Unsubscribe(dataConfig); + _dataAggregator.Remove(dataConfig); + } + + + /// + /// Removes the specified symbols to the subscription + /// + /// The symbols to be removed keyed by SecurityType + private bool Unsubscribe(IEnumerable symbols) + { + ProcessSubscriptionRequest(); + return true; + } + + /// + /// Returns whether the data provider is connected + /// + /// true if the data provider is connected + public bool IsConnected => true; + + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + public void Dispose() + { + _client.TradeEvent -= OnTrade; + _client.QuoteEvent -= OnQuote; + _client.Error -= OnError; + _client.Dispose(); + _dataAggregator.DisposeSafely(); + } + + /// + /// Helper method used in QC backend + /// + /// List of LEAN markets (exchanges) to subscribe + public void SubscribeMarkets(List markets) + { + Log.Trace($"CoinApiDataQueueHandler.SubscribeMarkets(): {string.Join(",", markets)}"); + + // we add '_' to be more precise, for example requesting 'BINANCE' doesn't match 'BINANCEUS' + SendHelloMessage(markets.Select(x => string.Concat(_symbolMapper.GetExchangeId(x.ToLowerInvariant()), "_"))); + } + + private void ProcessSubscriptionRequest() + { + if (_subscriptionsPending) return; + + _lastSubscribeRequestUtcTime = DateTime.UtcNow; + _subscriptionsPending = true; + + Task.Run(async () => + { + while (true) + { + DateTime requestTime; + List symbolsToSubscribe; + lock (_lockerSubscriptions) + { + requestTime = _lastSubscribeRequestUtcTime.Add(_subscribeDelay); + + // CoinAPI requires at least 5 seconds between hello messages + if (_nextHelloMessageUtcTime != DateTime.MinValue && requestTime < _nextHelloMessageUtcTime) + { + requestTime = _nextHelloMessageUtcTime; + } + + symbolsToSubscribe = _subscriptionManager.GetSubscribedSymbols().ToList(); + } + + var timeToWait = requestTime - DateTime.UtcNow; + + int delayMilliseconds; + if (timeToWait <= TimeSpan.Zero) + { + // minimum delay has passed since last subscribe request, send the Hello message + SubscribeSymbols(symbolsToSubscribe); + + lock (_lockerSubscriptions) + { + _lastSubscribeRequestUtcTime = DateTime.UtcNow; + if (_subscriptionManager.GetSubscribedSymbols().Count() == symbolsToSubscribe.Count) + { + // no more subscriptions pending, task finished + _subscriptionsPending = false; + break; + } + } + + delayMilliseconds = _subscribeDelay.Milliseconds; + } + else + { + delayMilliseconds = timeToWait.Milliseconds; + } + + await Task.Delay(delayMilliseconds).ConfigureAwait(false); + } + }); + } + + /// + /// Returns true if we can subscribe to the specified symbol + /// + private static bool CanSubscribe(Symbol symbol) + { + // ignore unsupported security types + if (symbol.ID.SecurityType != SecurityType.Crypto && symbol.ID.SecurityType != SecurityType.CryptoFuture) + { + return false; + } + + // ignore universe symbols + return !symbol.Value.Contains("-UNIVERSE-"); + } + + /// + /// Subscribes to a list of symbols + /// + /// The list of symbols to subscribe + private void SubscribeSymbols(List symbolsToSubscribe) + { + Log.Trace($"CoinApiDataQueueHandler.SubscribeSymbols(): {string.Join(",", symbolsToSubscribe)}"); + + // subscribe to symbols using exact match + SendHelloMessage(symbolsToSubscribe.Select(x => + { + try + { + var result = string.Concat(_symbolMapper.GetBrokerageSymbol(x), "$"); + return result; + } + catch (Exception e) + { + Log.Error(e); + return null; + } + }).Where(x => x != null)); + } + + private void SendHelloMessage(IEnumerable subscribeFilter) + { + var list = subscribeFilter.ToList(); + if (list.Count == 0) + { + // If we use a null or empty filter in the CoinAPI hello message + // we will be subscribing to all symbols for all active exchanges! + // Only option is requesting an invalid symbol as filter. + list.Add("$no_symbol_requested$"); + } + + _client.SendHelloMessage(new Hello + { + apikey = Guid.Parse(_apiKey), + heartbeat = true, + subscribe_data_type = _streamingDataType, + subscribe_filter_symbol_id = list.ToArray() + }); + + _nextHelloMessageUtcTime = DateTime.UtcNow.Add(_minimumTimeBetweenHelloMessages); + } + + private void OnTrade(object sender, Trade trade) + { + try + { + var symbol = GetSymbolUsingCache(trade.symbol_id); + if (symbol == null) + { + return; + } + + var tick = new Tick(trade.time_exchange, symbol, string.Empty, string.Empty, quantity: trade.size, price: trade.price); + + lock (symbol) + { + _dataAggregator.Update(tick); + } + } + catch (Exception e) + { + Log.Error(e); + } + } + + private void OnQuote(object sender, Quote quote) + { + try + { + // only emit quote ticks if bid price or ask price changed + Tick previousQuote; + if (!_previousQuotes.TryGetValue(quote.symbol_id, out previousQuote) + || quote.ask_price != previousQuote.AskPrice + || quote.bid_price != previousQuote.BidPrice) + { + var symbol = GetSymbolUsingCache(quote.symbol_id); + if (symbol == null) + { + return; + } + + var tick = new Tick(quote.time_exchange, symbol, string.Empty, string.Empty, + bidSize: quote.bid_size, bidPrice: quote.bid_price, + askSize: quote.ask_size, askPrice: quote.ask_price); + + _previousQuotes[quote.symbol_id] = tick; + lock (symbol) + { + _dataAggregator.Update(tick); + } + } + } + catch (Exception e) + { + Log.Error(e); + } + } + + private Symbol GetSymbolUsingCache(string ticker) + { + if (!_symbolCache.TryGetValue(ticker, out Symbol result)) + { + try + { + var securityType = ticker.IndexOf("_PERP_") > 0 ? SecurityType.CryptoFuture : SecurityType.Crypto; + result = _symbolMapper.GetLeanSymbol(ticker, securityType, string.Empty); + } + catch (Exception e) + { + Log.Error(e); + // we store the null so we don't keep going into the same mapping error + result = null; + } + _symbolCache[ticker] = result; + } + return result; + } + + private void OnError(object? sender, Exception e) + { + Log.Error(e); + } + + private class ModulesReadLicenseRead : Api.RestResponse + { + [JsonProperty(PropertyName = "license")] + public string License; + + [JsonProperty(PropertyName = "organizationId")] + public string OrganizationId; + } + + /// + /// Validate the user of this project has permission to be using it via our web API. + /// + private static void ValidateSubscription() + { + try + { + const int productId = 335; + var userId = Config.GetInt("job-user-id"); + var token = Config.Get("api-access-token"); + var organizationId = Config.Get("job-organization-id", null); + // Verify we can authenticate with this user and token + var api = new ApiConnection(userId, token); + if (!api.Connected) + { + throw new ArgumentException("Invalid api user id or token, cannot authenticate subscription."); + } + // Compile the information we want to send when validating + var information = new Dictionary() + { + {"productId", productId}, + {"machineName", Environment.MachineName}, + {"userName", Environment.UserName}, + {"domainName", Environment.UserDomainName}, + {"os", Environment.OSVersion} + }; + // IP and Mac Address Information + try + { + var interfaceDictionary = new List>(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces().Where(nic => nic.OperationalStatus == OperationalStatus.Up)) + { + var interfaceInformation = new Dictionary(); + // Get UnicastAddresses + var addresses = nic.GetIPProperties().UnicastAddresses + .Select(uniAddress => uniAddress.Address) + .Where(address => !IPAddress.IsLoopback(address)).Select(x => x.ToString()); + // If this interface has non-loopback addresses, we will include it + if (!addresses.IsNullOrEmpty()) + { + interfaceInformation.Add("unicastAddresses", addresses); + // Get MAC address + interfaceInformation.Add("MAC", nic.GetPhysicalAddress().ToString()); + // Add Interface name + interfaceInformation.Add("name", nic.Name); + // Add these to our dictionary + interfaceDictionary.Add(interfaceInformation); + } + } + information.Add("networkInterfaces", interfaceDictionary); + } + catch (Exception) + { + // NOP, not necessary to crash if fails to extract and add this information + } + // Include our OrganizationId if specified + if (!string.IsNullOrEmpty(organizationId)) + { + information.Add("organizationId", organizationId); + } + var request = new RestRequest("modules/license/read", Method.POST) { RequestFormat = DataFormat.Json }; + request.AddParameter("application/json", JsonConvert.SerializeObject(information), ParameterType.RequestBody); + api.TryRequest(request, out ModulesReadLicenseRead result); + if (!result.Success) + { + throw new InvalidOperationException($"Request for subscriptions from web failed, Response Errors : {string.Join(',', result.Errors)}"); + } + + var encryptedData = result.License; + // Decrypt the data we received + DateTime? expirationDate = null; + long? stamp = null; + bool? isValid = null; + if (encryptedData != null) + { + // Fetch the org id from the response if it was not set, we need it to generate our validation key + if (string.IsNullOrEmpty(organizationId)) + { + organizationId = result.OrganizationId; + } + // Create our combination key + var password = $"{token}-{organizationId}"; + var key = SHA256.HashData(Encoding.UTF8.GetBytes(password)); + // Split the data + var info = encryptedData.Split("::"); + var buffer = Convert.FromBase64String(info[0]); + var iv = Convert.FromBase64String(info[1]); + // Decrypt our information + using var aes = new AesManaged(); + var decryptor = aes.CreateDecryptor(key, iv); + using var memoryStream = new MemoryStream(buffer); + using var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read); + using var streamReader = new StreamReader(cryptoStream); + var decryptedData = streamReader.ReadToEnd(); + if (!decryptedData.IsNullOrEmpty()) + { + var jsonInfo = JsonConvert.DeserializeObject(decryptedData); + expirationDate = jsonInfo["expiration"]?.Value(); + isValid = jsonInfo["isValid"]?.Value(); + stamp = jsonInfo["stamped"]?.Value(); + } + } + // Validate our conditions + if (!expirationDate.HasValue || !isValid.HasValue || !stamp.HasValue) + { + throw new InvalidOperationException("Failed to validate subscription."); + } + + var nowUtc = DateTime.UtcNow; + var timeSpan = nowUtc - Time.UnixTimeStampToDateTime(stamp.Value); + if (timeSpan > TimeSpan.FromHours(12)) + { + throw new InvalidOperationException("Invalid API response."); + } + if (!isValid.Value) + { + throw new ArgumentException($"Your subscription is not valid, please check your product subscriptions on our website."); + } + if (expirationDate < nowUtc) + { + throw new ArgumentException($"Your subscription expired {expirationDate}, please renew in order to use this product."); + } + } + catch (Exception e) + { + Log.Error($"{nameof(CoinApiDataQueueHandler)}.{nameof(ValidateSubscription)}: Failed during validation, shutting down. Error : {e.Message}"); + throw; + } + } + } +} diff --git a/QuantConnect.CoinAPI/CoinApiProduct.cs b/QuantConnect.CoinAPI/CoinApiProduct.cs new file mode 100644 index 0000000..1055342 --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApiProduct.cs @@ -0,0 +1,49 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.CoinAPI +{ + /// + /// Coin API's available tariff plans (or products). + /// https://www.coinapi.io/Pricing + /// + public enum CoinApiProduct + { + /// + /// 100 daily requests, trades test only + /// + Free, + + /// + /// 1k daily requests, trades only + /// + Startup, + + /// + /// 10k daily requests, trades + quotes + /// + Streamer, + + /// + /// 100k daily requests, unlimited websocket + /// + Professional, + + /// + /// Contact Coin Api sales for more info + /// + Enterprise + } +} diff --git a/QuantConnect.CoinAPI/CoinApiSymbol.cs b/QuantConnect.CoinAPI/CoinApiSymbol.cs new file mode 100644 index 0000000..0f56083 --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApiSymbol.cs @@ -0,0 +1,42 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI +{ + public class CoinApiSymbol + { + [JsonProperty("symbol_id")] + public string SymbolId { get; set; } + + [JsonProperty("exchange_id")] + public string ExchangeId { get; set; } + + [JsonProperty("symbol_type")] + public string SymbolType { get; set; } + + [JsonProperty("asset_id_base")] + public string AssetIdBase { get; set; } + + [JsonProperty("asset_id_quote")] + public string AssetIdQuote { get; set; } + + public override string ToString() + { + return SymbolId; + } + } +} diff --git a/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs new file mode 100644 index 0000000..8a3d94e --- /dev/null +++ b/QuantConnect.CoinAPI/CoinApiSymbolMapper.cs @@ -0,0 +1,266 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; +using QuantConnect.Logging; +using QuantConnect.Securities; +using QuantConnect.Brokerages; +using QuantConnect.Configuration; +using QuantConnect.CoinAPI.Models; + +namespace QuantConnect.CoinAPI +{ + /// + /// Provides the mapping between Lean symbols and CoinAPI symbols. + /// + /// For now we only support mapping for CoinbasePro (GDAX) and Bitfinex + public class CoinApiSymbolMapper : ISymbolMapper + { + private const string RestUrl = "https://rest.coinapi.io"; + private readonly string _apiKey = Config.Get("coinapi-api-key"); + private readonly bool _useLocalSymbolList = Config.GetBool("coinapi-use-local-symbol-list"); + + private readonly FileInfo _coinApiSymbolsListFile = new FileInfo( + Config.Get("coinapi-default-symbol-list-file", "CoinApiSymbols.json")); + /// + /// LEAN market <-> CoinAPI exchange id maps + /// + /// + public static readonly Dictionary MapMarketsToExchangeIds = new Dictionary + { + { Market.Coinbase, "COINBASE" }, + { Market.Bitfinex, "BITFINEX" }, + { Market.Binance, "BINANCE" }, + { Market.Kraken, "KRAKEN" }, + { Market.BinanceUS, "BINANCEUS" }, + }; + private static readonly Dictionary MapExchangeIdsToMarkets = + MapMarketsToExchangeIds.ToDictionary(x => x.Value, x => x.Key); + + private static readonly Dictionary> CoinApiToLeanCurrencyMappings = + new Dictionary> + { + { + Market.Bitfinex, + new Dictionary + { + { "ABS", "ABYSS"}, + { "AIO", "AION"}, + { "ALG", "ALGO"}, + { "AMP", "AMPL"}, + { "ATO", "ATOM"}, + { "BCHABC", "BCH"}, + { "BCHSV", "BSV"}, + { "CSX", "CS"}, + { "CTX", "CTXC"}, + { "DOG", "MDOGE"}, + { "DRN", "DRGN"}, + { "DTX", "DT"}, + { "EDO", "PNT"}, + { "EUS", "EURS"}, + { "EUT", "EURT"}, + { "GSD", "GUSD"}, + { "HOPL", "HOT"}, + { "IOS", "IOST"}, + { "IOT", "IOTA"}, + { "LOO", "LOOM"}, + { "MIT", "MITH"}, + { "NCA", "NCASH"}, + { "OMN", "OMNI"}, + { "ORS", "ORST"}, + { "PAS", "PASS"}, + { "PKGO", "GOT"}, + { "POY", "POLY"}, + { "QSH", "QASH"}, + { "REP", "REP2"}, + { "SCR", "XD"}, + { "SNG", "SNGLS"}, + { "SPK", "SPANK"}, + { "STJ", "STORJ"}, + { "TSD", "TUSD"}, + { "UDC", "USDC"}, + { "ULTRA", "UOS"}, + { "USK", "USDK"}, + { "UTN", "UTNP"}, + { "VSY", "VSYS"}, + { "WBT", "WBTC"}, + { "XCH", "XCHF"}, + { "YGG", "YEED"} + } + } + }; + + // map LEAN symbols to CoinAPI symbol ids + private Dictionary _symbolMap = new Dictionary(); + + + /// + /// Creates a new instance of the class + /// + public CoinApiSymbolMapper() + { + MapExchangeIdsToMarkets["BINANCEFTS"] = Market.Binance; + MapExchangeIdsToMarkets["BINANCEFTSC"] = Market.Binance; + + MapExchangeIdsToMarkets["BYBITSPOT"] = Market.Bybit; + + LoadSymbolMap(MapExchangeIdsToMarkets.Keys.ToArray()); + } + + /// + /// Converts a Lean symbol instance to a CoinAPI symbol id + /// + /// A Lean symbol instance + /// The CoinAPI symbol id + public string GetBrokerageSymbol(Symbol symbol) + { + if (!_symbolMap.TryGetValue(symbol, out var symbolId)) + { + throw new Exception($"CoinApiSymbolMapper.GetBrokerageSymbol(): Symbol not found: {symbol}"); + } + + return symbolId; + } + + /// + /// Converts a CoinAPI symbol id to a Lean symbol instance + /// + /// The CoinAPI symbol id + /// The security type + /// The market + /// Expiration date of the security (if applicable) + /// The strike of the security (if applicable) + /// The option right of the security (if applicable) + /// A new Lean Symbol instance + public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, + DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = OptionRight.Call) + { + var parts = brokerageSymbol.Split('_'); + if (parts.Length != 4) + { + throw new Exception($"CoinApiSymbolMapper.GetLeanSymbol(): Unsupported SymbolId: {brokerageSymbol}"); + } + + string symbolMarket; + if (!MapExchangeIdsToMarkets.TryGetValue(parts[0], out symbolMarket)) + { + throw new Exception($"CoinApiSymbolMapper.GetLeanSymbol(): Unsupported ExchangeId: {parts[0]}"); + } + + var baseCurrency = ConvertCoinApiCurrencyToLeanCurrency(parts[2], symbolMarket); + var quoteCurrency = ConvertCoinApiCurrencyToLeanCurrency(parts[3], symbolMarket); + + var ticker = baseCurrency + quoteCurrency; + + return Symbol.Create(ticker, securityType, symbolMarket); + } + + /// + /// Returns the CoinAPI exchange id for the given market + /// + /// The Lean market + /// The CoinAPI exchange id + public string GetExchangeId(string market) + { + string exchangeId; + MapMarketsToExchangeIds.TryGetValue(market, out exchangeId); + + return exchangeId; + } + + private void LoadSymbolMap(string[] exchangeIds) + { + var list = string.Join(",", exchangeIds); + var json = string.Empty; + + if (_useLocalSymbolList) + { + if (!_coinApiSymbolsListFile.Exists) + { + throw new Exception($"CoinApiSymbolMapper.LoadSymbolMap(): File not found: {_coinApiSymbolsListFile.FullName}, please " + + $"download the latest symbol list from CoinApi."); + } + json = File.ReadAllText(_coinApiSymbolsListFile.FullName); + } + else + { + json = $"{RestUrl}/v1/symbols?filter_symbol_id={list}&apiKey={_apiKey}".DownloadData(); + } + + List? result = new(); + + try + { + result = JsonConvert.DeserializeObject>(json); + } + catch (JsonSerializationException) + { + var error = JsonConvert.DeserializeObject(json); + throw new Exception(error.Error); + } + + // There were cases of entries in the CoinApiSymbols list with the following pattern: + // _SPOT___ + // Those cases should be ignored for SPOT prices. + foreach (var x in result + .Where(x => x.SymbolId.Split('_').Length == 4 && + // exclude Bitfinex BCH pre-2018-fork as for now we don't have historical mapping data + (x.ExchangeId != "BITFINEX" || x.AssetIdBase != "BCH" && x.AssetIdQuote != "BCH") + // solves the cases where we request 'binance' and get 'binanceus' + && MapExchangeIdsToMarkets.ContainsKey(x.ExchangeId))) + { + var market = MapExchangeIdsToMarkets[x.ExchangeId]; + + SecurityType securityType; + if (x.SymbolType == "SPOT") + { + securityType = SecurityType.Crypto; + } + else if (x.SymbolType == "PERPETUAL") + { + securityType = SecurityType.CryptoFuture; + } + else + { + continue; + } + var symbol = GetLeanSymbol(x.SymbolId, securityType, market); + + if (_symbolMap.ContainsKey(symbol)) + { + // skipping duplicate symbols. Kraken has both USDC/AD & USD/CAD symbols + Log.Error($"CoinApiSymbolMapper(): Duplicate symbol found {symbol} will be skipped!"); + continue; + } + _symbolMap[symbol] = x.SymbolId; + } + } + + private static string ConvertCoinApiCurrencyToLeanCurrency(string currency, string market) + { + Dictionary mappings; + if (CoinApiToLeanCurrencyMappings.TryGetValue(market, out mappings)) + { + string mappedCurrency; + if (mappings.TryGetValue(currency, out mappedCurrency)) + { + currency = mappedCurrency; + } + } + + return currency; + } + } +} diff --git a/QuantConnect.CoinAPI/Messages/BaseMessage.cs b/QuantConnect.CoinAPI/Messages/BaseMessage.cs new file mode 100644 index 0000000..a168252 --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/BaseMessage.cs @@ -0,0 +1,25 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class BaseMessage + { + [JsonProperty("type")] + public string Type { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/ErrorMessage.cs b/QuantConnect.CoinAPI/Messages/ErrorMessage.cs new file mode 100644 index 0000000..924c972 --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/ErrorMessage.cs @@ -0,0 +1,25 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class ErrorMessage : BaseMessage + { + [JsonProperty("message")] + public string Message { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/HelloMessage.cs b/QuantConnect.CoinAPI/Messages/HelloMessage.cs new file mode 100644 index 0000000..cd79290 --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/HelloMessage.cs @@ -0,0 +1,40 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class HelloMessage + { + [JsonProperty("type")] + public string Type { get; } = "hello"; + + [JsonProperty("apikey")] + public string ApiKey { get; set; } + + [JsonProperty("heartbeat")] + public bool Heartbeat { get; set; } + + [JsonProperty("subscribe_data_type")] + public string[] SubscribeDataType { get; set; } + + [JsonProperty("subscribe_filter_symbol_id")] + public string[] SubscribeFilterSymbolId { get; set; } + + [JsonProperty("subscribe_filter_asset_id")] + public string[] SubscribeFilterAssetId { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs b/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs new file mode 100644 index 0000000..ce5a83a --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/HistoricalDataMessage.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class HistoricalDataMessage + { + [JsonProperty("time_period_start")] + public DateTime TimePeriodStart { get; set; } + + [JsonProperty("time_period_end")] + public DateTime TimePeriodEnd { get; set; } + + [JsonProperty("price_open")] + public decimal PriceOpen { get; set; } + + [JsonProperty("price_high")] + public decimal PriceHigh { get; set; } + + [JsonProperty("price_low")] + public decimal PriceLow { get; set; } + + [JsonProperty("price_close")] + public decimal PriceClose { get; set; } + + [JsonProperty("volume_traded")] + public decimal VolumeTraded { get; set; } + + [JsonProperty("trades_count")] + public int TradesCount { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/QuoteMessage.cs b/QuantConnect.CoinAPI/Messages/QuoteMessage.cs new file mode 100644 index 0000000..59c273c --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/QuoteMessage.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class QuoteMessage : BaseMessage + { + [JsonProperty("symbol_id")] + public string SymbolId { get; set; } + + [JsonProperty("sequence")] + public long Sequence { get; set; } + + [JsonProperty("time_exchange")] + public DateTime TimeExchange { get; set; } + + [JsonProperty("time_coinapi")] + public DateTime TimeCoinApi { get; set; } + + [JsonProperty("ask_price")] + public decimal AskPrice { get; set; } + + [JsonProperty("ask_size")] + public decimal AskSize { get; set; } + + [JsonProperty("bid_price")] + public decimal BidPrice { get; set; } + + [JsonProperty("bid_size")] + public decimal BidSize { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Messages/TradeMessage.cs b/QuantConnect.CoinAPI/Messages/TradeMessage.cs new file mode 100644 index 0000000..cfa6b31 --- /dev/null +++ b/QuantConnect.CoinAPI/Messages/TradeMessage.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Messages +{ + public class TradeMessage : BaseMessage + { + [JsonProperty("symbol_id")] + public string SymbolId { get; set; } + + [JsonProperty("sequence")] + public long Sequence { get; set; } + + [JsonProperty("time_exchange")] + public DateTime TimeExchange { get; set; } + + [JsonProperty("time_coinapi")] + public DateTime TimeCoinApi { get; set; } + + [JsonProperty("uuid")] + public string Uuid { get; set; } + + [JsonProperty("price")] + public decimal Price { get; set; } + + [JsonProperty("size")] + public decimal Size { get; set; } + + [JsonProperty("taker_side")] + public string TakerSide { get; set; } + } +} diff --git a/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs b/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs new file mode 100644 index 0000000..56133b1 --- /dev/null +++ b/QuantConnect.CoinAPI/Models/CoinApiErrorResponse.cs @@ -0,0 +1,31 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.CoinAPI.Models +{ + public readonly struct CoinApiErrorResponse + { + [JsonProperty("error")] + public string Error { get; } + + [JsonConstructor] + public CoinApiErrorResponse(string error) + { + Error = error; + } + } +} diff --git a/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj b/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj new file mode 100644 index 0000000..3096705 --- /dev/null +++ b/QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj @@ -0,0 +1,40 @@ + + + + Release + AnyCPU + net6.0 + QuantConnect.CoinAPI + QuantConnect.CoinAPI + QuantConnect.CoinAPI + QuantConnect.CoinAPI + Library + bin\$(Configuration) + false + true + false + QuantConnect LEAN CoinAPI Data Source: CoinAPI Data Source plugin for Lean + enable + enable + + + + full + bin\Debug\ + + + + pdbonly + bin\Release\ + + + + + + + + + + + + diff --git a/QuantConnect.DataSource.csproj b/QuantConnect.DataSource.csproj deleted file mode 100644 index f379576..0000000 --- a/QuantConnect.DataSource.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - net6.0 - QuantConnect.DataSource - QuantConnect.DataSource.MyCustomDataType - bin\$(Configuration) - $(OutputPath)\QuantConnect.DataSource.MyCustomDataType.xml - - - - - - - - - - - - - - - - - - - - diff --git a/README.md b/README.md index 642fc49..abf0d10 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,60 @@ ![LEAN Data Source SDK](http://cdn.quantconnect.com.s3.us-east-1.amazonaws.com/datasources/Github_LeanDataSourceSDK.png) -# Lean DataSource SDK +# Lean CoinAPI DataSource Plugin [![Build Status](https://github.com/QuantConnect/LeanDataSdk/workflows/Build%20%26%20Test/badge.svg)](https://github.com/QuantConnect/LeanDataSdk/actions?query=workflow%3A%22Build%20%26%20Test%22) ### Introduction -The Lean Data SDK is a cross-platform template repository for developing custom data types for Lean. -These data types will be consumed by [QuantConnect](https://www.quantconnect.com/) trading algorithms and research environment, locally or in the cloud. +Welcome to the CoinAPI Connector Library for .NET 6. This open-source project provides a robust and efficient C# library designed to seamlessly connect with the CoinAPI. The library facilitates easy integration with the QuantConnect [LEAN Algorithmic Trading Engine](https://github.com/quantConnect/Lean), offering a clear and straightforward way for users to incorporate CoinAPI's extensive financial datasets into their algorithmic trading strategies. -It is composed by example .Net solution for the data type and converter scripts. +### CoinAPI Overview +CoinAPI is a reliable provider of real-time and historical financial market data, offering support for traditional asset classes such as cryptocurrencies and crypto futures across various exchanges. With CoinAPI, developers can access a wealth of data to enhance their trading strategies and decision-making processes. -### Prerequisites +### Features -The solution targets dotnet 5, for installation instructions please follow [dotnet download](https://dotnet.microsoft.com/download). +- **Easy Integration:** Simple and intuitive integration process, allowing developers to quickly incorporate CoinAPI's data into their trading algorithms. +- **Rich Financial Data:** Access to a vast array of real-time and historical data for cryptocurrencies and crypto futures, empowering developers to make informed trading decisions. +- **Flexible Configuration:** Customizable settings to tailor the integration according to specific trading needs and preferences. +- **Symbol SecurityType Support:** + - [x] Crypto + - [x] CryptoFuture +- **Exchange Support:** + - [x] COINBASE + - [x] BITFINEX + - [x] BINANCE + - [x] KRAKEN + - [x] BINANCEUS +- **Backtesting and Research:** Seamlessly test and refine your trading algorithms using CoinAPI's data within QuantConnect LEAN's backtesting and research modes, enabling you to optimize your strategies with confidence. -The data downloader and converter script can be developed in different ways: C# executable, Python script, Python Jupyter notebook or even a bash script. -- The python script should be compatible with python 3.6.8 -- Bash script will run on Ubuntu Bionic - -Specifically, the enviroment where these scripts will be run is [quantconnect/research](https://hub.docker.com/repository/docker/quantconnect/research) based on [quantconnect/lean:foundation](https://hub.docker.com/repository/docker/quantconnect/lean). +### Contribute to the Project +Contributions to this open-source project are welcome! If you find any issues, have suggestions for improvements, or want to add new features, please open an issue or submit a pull request. ### Installation - -The "Use this template" feature should be used for each unique data source which requires its own data processing. Once it is cloned locally, you should be able to successfully build the solution, run all tests and execute the downloader and/or conveter scripts. The final version should pass all CI tests of GitHub Actions. - -Once ready, please contact support@quantconnect.com and we will create a listing in the QuantConnect Data Market for your company and link to your public repository and commit hash. - -### Datasets Vendor Requirements - -Key requirements for new vendors include: - - - A well-defined dataset with a clear and static vision for the data to minimize churn or changes as people will be building systems from it. This is easiest with "raw" data (e.g. sunshine hours vs a sentiment algorithm) - - Robust ticker and security links to ensure the tickers are tracked well through time, or accurately point in time. ISIN, FIGI, or point in time ticker supported - - Robust funding to ensure viable for at least 1 year - - Robust API to ensure reliable up-time. No dead links on site or and 502 servers while using API - - Consistent delivery schedule, on time and in time for market trading - - Consistent data format with notifications and lead time on data format updates - - At least 1 year of historical point in time data - - Survivorship bias free data - - Good documentation for the dataset - - -### Tutorials - - - See [Tutorials](https://www.quantconnect.com/docs/v2/our-platform/datasets/contributing-datasets) for a step by step guide for creating a new LEAN Data Source. \ No newline at end of file +To contribute to the CoinAPI Connector Library for .NET 6 within QuantConnect LEAN, follow these steps: +1. **Obtain API Key:** Sign up for a free CoinAPI key [here](https://docs.coinapi.io/) if you don't have one. +2. **Fork the Project:** Fork the repository by clicking the "Fork" button at the top right of the GitHub page. +3. Clone Your Forked Repository: +``` +git clone https://github.com/your-username/Lean.DataSource.CoinAPI.git +``` +4. **Configuration:** + - Set the `coinapi-api-key` in your QuantConnect configuration (config.json or environment variables). + - [optional] Set the `coinapi-product` (by default: Free) +``` +{ + "coinapi-api-key": "", + "coinapi-product": "", +} +``` + +### Price Plan +For detailed information on CoinAPI's pricing plans, please refer to the [CoinAPI Pricing](https://www.coinapi.io/market-data-api/pricing) page. + +### Documentation +Refer to the [documentation](https://www.quantconnect.com/docs/v2/lean-cli/datasets/coinapi) for detailed information on the library's functions, parameters, and usage examples. + +### License +This project is licensed under the MIT License - see the [LICENSE](#) file for details. + +Happy coding and algorithmic trading! \ No newline at end of file diff --git a/examples.md b/examples.md deleted file mode 100644 index 9086ae3..0000000 --- a/examples.md +++ /dev/null @@ -1 +0,0 @@ -https://github.com/QuantConnect?q=Lean.DataSource&type=&language=&sort= \ No newline at end of file diff --git a/output/alternative/mycustomdatatype/spy.csv b/output/alternative/mycustomdatatype/spy.csv deleted file mode 100644 index e450a4d..0000000 --- a/output/alternative/mycustomdatatype/spy.csv +++ /dev/null @@ -1,6 +0,0 @@ -20131001,buy -20131003,buy -20131006,buy -20131007,sell -20131009,buy -20131011,sell \ No newline at end of file diff --git a/renameDataset.sh b/renameDataset.sh deleted file mode 100644 index 51ccd6d..0000000 --- a/renameDataset.sh +++ /dev/null @@ -1,57 +0,0 @@ -# Get {vendorNameDatasetName} -vendorNameDatasetName=${PWD##*.} -vendorNameDatasetNameUniverse=${vendorNameDatasetName}Universe - -# Rename the MyCustomDataType.cs file to {vendorNameDatasetName}.cs -mv MyCustomDataType.cs ${vendorNameDatasetName}.cs -mv MyCustomDataUniverseType.cs ${vendorNameDatasetNameUniverse}.cs - -# In the QuantConnect.DataSource.csproj file, rename the MyCustomDataType class to {vendorNameDatasetName} -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" QuantConnect.DataSource.csproj -sed -i "s/Demonstration.cs/${vendorNameDatasetName}Algorithm.cs/g" QuantConnect.DataSource.csproj -sed -i "s/DemonstrationUniverse.cs/${vendorNameDatasetNameUniverse}SelectionAlgorithm.cs/g" QuantConnect.DataSource.csproj - -# In the {vendorNameDatasetName}.cs file, rename the MyCustomDataType class to {vendorNameDatasetName} -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" ${vendorNameDatasetName}.cs - -# In the {vendorNameDatasetNameUniverse}.cs file, rename the MyCustomDataUniverseType class to {vendorNameDatasetNameUniverse} -sed -i "s/MyCustomDataUniverseType/$vendorNameDatasetNameUniverse/g" ${vendorNameDatasetNameUniverse}.cs - -# In the {vendorNameDatasetName}Algorithm.cs file, rename the MyCustomDataType class to to {vendorNameDatasetName} -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" Demonstration.cs -sed -i "s/MyCustomDataType/$vendorNameDatasetName/g" Demonstration.py - -# In the {vendorNameDatasetName}Algorithm.cs file, rename the CustomDataAlgorithm class to {vendorNameDatasetName}Algorithm -sed -i "s/CustomDataAlgorithm/${vendorNameDatasetName}Algorithm/g" Demonstration.cs -sed -i "s/CustomDataAlgorithm/${vendorNameDatasetName}Algorithm/g" Demonstration.py - -# In the {vendorNameDatasetName}UniverseSelectionAlgorithm.cs file, rename the MyCustomDataUniverseType class to to {vendorNameDatasetName}Universe -sed -i "s/MyCustomDataUniverseType/$vendorNameDatasetNameUniverse/g" DemonstrationUniverse.cs -sed -i "s/MyCustomDataUniverseType/$vendorNameDatasetNameUniverse/g" DemonstrationUniverse.py - -# In the {vendorNameDatasetNameUniverse}SelectionAlgorithm.cs file, rename the CustomDataAlgorithm class to {vendorNameDatasetNameUniverse}SelectionAlgorithm -sed -i "s/CustomDataUniverse/${vendorNameDatasetNameUniverse}SelectionAlgorithm/g" DemonstrationUniverse.cs -sed -i "s/CustomDataUniverse/${vendorNameDatasetNameUniverse}SelectionAlgorithm/g" DemonstrationUniverse.py - -# Rename the Lean.DataSource.vendorNameDatasetName/Demonstration.cs/py file to {vendorNameDatasetName}Algorithm.cs/py -mv Demonstration.cs ${vendorNameDatasetName}Algorithm.cs -mv Demonstration.py ${vendorNameDatasetName}Algorithm.py - -# Rename the Lean.DataSource.vendorNameDatasetName/DemonstrationUniverseSelectionAlgorithm.cs/py file to {vendorNameDatasetName}UniverseSelectionAlgorithm.cs/py -mv DemonstrationUniverse.cs ${vendorNameDatasetNameUniverse}SelectionAlgorithm.cs -mv DemonstrationUniverse.py ${vendorNameDatasetNameUniverse}SelectionAlgorithm.py - -# Rename the tests/MyCustomDataTypeTests.cs file to tests/{vendorNameDatasetName}Tests.cs -sed -i "s/MyCustomDataType/${vendorNameDatasetName}/g" tests/MyCustomDataTypeTests.cs -mv tests/MyCustomDataTypeTests.cs tests/${vendorNameDatasetName}Tests.cs - -# In tests/Tests.csproj, rename the Demonstration.cs and DemonstrationUniverse.cs to {vendorNameDatasetName}Algorithm.cs and {vendorNameDatasetNameUniverse}SelectionAlgorithm.cs -sed -i "s/Demonstration.cs/${vendorNameDatasetName}Algorithm.cs/g" tests/Tests.csproj -sed -i "s/DemonstrationUniverse.cs/${vendorNameDatasetNameUniverse}SelectionAlgorithm.cs/g" tests/Tests.csproj - -# In the MyCustomDataDownloader.cs and Program.cs files, rename the MyCustomDataDownloader to {vendorNameDatasetNameUniverse}DataDownloader -sed -i "s/MyCustomDataDownloader/${vendorNameDatasetNameUniverse}DataDownloader/g" DataProcessing/Program.cs -sed -i "s/MyCustomDataDownloader/${vendorNameDatasetNameUniverse}DataDownloader/g" DataProcessing/MyCustomDataDownloader.cs - -# Rename the DataProcessing/MyCustomDataDownloader.cs file to DataProcessing/{vendorNameDatasetName}DataDownloader.cs -mv DataProcessing/MyCustomDataDownloader.cs DataProcessing/${vendorNameDatasetName}DataDownloader.cs \ No newline at end of file diff --git a/tests/MyCustomDataTypeTests.cs b/tests/MyCustomDataTypeTests.cs deleted file mode 100644 index c0b907c..0000000 --- a/tests/MyCustomDataTypeTests.cs +++ /dev/null @@ -1,99 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using ProtoBuf; -using System.IO; -using System.Linq; -using ProtoBuf.Meta; -using Newtonsoft.Json; -using NUnit.Framework; -using QuantConnect.Data; -using QuantConnect.DataSource; - -namespace QuantConnect.DataLibrary.Tests -{ - [TestFixture] - public class MyCustomDataTypeTests - { - [Test] - public void JsonRoundTrip() - { - var expected = CreateNewInstance(); - var type = expected.GetType(); - var serialized = JsonConvert.SerializeObject(expected); - var result = JsonConvert.DeserializeObject(serialized, type); - - AssertAreEqual(expected, result); - } - - [Test] - public void ProtobufRoundTrip() - { - var expected = CreateNewInstance(); - var type = expected.GetType(); - - RuntimeTypeModel.Default[typeof(BaseData)].AddSubType(2000, type); - - using (var stream = new MemoryStream()) - { - Serializer.Serialize(stream, expected); - - stream.Position = 0; - - var result = Serializer.Deserialize(type, stream); - - AssertAreEqual(expected, result, filterByCustomAttributes: true); - } - } - - [Test] - public void Clone() - { - var expected = CreateNewInstance(); - var result = expected.Clone(); - - AssertAreEqual(expected, result); - } - - private void AssertAreEqual(object expected, object result, bool filterByCustomAttributes = false) - { - foreach (var propertyInfo in expected.GetType().GetProperties()) - { - // we skip Symbol which isn't protobuffed - if (filterByCustomAttributes && propertyInfo.CustomAttributes.Count() != 0) - { - Assert.AreEqual(propertyInfo.GetValue(expected), propertyInfo.GetValue(result)); - } - } - foreach (var fieldInfo in expected.GetType().GetFields()) - { - Assert.AreEqual(fieldInfo.GetValue(expected), fieldInfo.GetValue(result)); - } - } - - private BaseData CreateNewInstance() - { - return new MyCustomDataType - { - Symbol = Symbol.Empty, - Time = DateTime.Today, - DataType = MarketDataType.Base, - SomeCustomProperty = "This is some market related information" - }; - } - } -} \ No newline at end of file diff --git a/tests/Tests.csproj b/tests/Tests.csproj deleted file mode 100644 index 8e308d0..0000000 --- a/tests/Tests.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - net6.0 - QuantConnect.DataLibrary.Tests - - - - - - - - - - all - - - - - - - - -