update to all channels

mediathekview · Jun 7, 2024 · 0a4e8c9 · 0a4e8c9
1 parent df8ef07
commit 0a4e8c9
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 61 deletions.
diff --git a/config/Test-MServer-Config-R1.yaml → config/MServer-Config-R1.yaml b/config/Test-MServer-Config-R1.yaml → config/MServer-Config-R1.yaml
@@ -17,22 +17,22 @@ maximumRequestsPerSecond: 999.0
 
 # If set only these Sender will be crawled all other will be ignored.
 senderIncluded:
-  #- ARD
+  - ARD
   #- ARTE_DE
-  #- ARGE_FR
+  #- ARTE_FR
   #- ARTE_EN
   #- ARTE_PL
   #- ARTE_IT
   #- ARTE_ES
-  #- DREISAT
-  #- FUNK
-  #- KIKA
+  - DREISAT
+  - FUNK
+  - KIKA
   - DW
-  #- ORF
-  #- PHOENIX
-  #- SRF
-  #- SR
-  #- ZDF
+  - ORF
+  - PHOENIX
+  - SRF
+  - SR
+  - ZDF
 
 # If set the server will be awake after the crawler run and restarts the run after the given amount.
 #schedules:
@@ -64,7 +64,7 @@ filmlistSaveFormats:
 # The paths where which filmlist should be safed to.
 filmlistSavePaths:
 #  JSON: target/filmlists/filmliste.json
-  OLD_JSON: target/filmlists/filmliste_old.json
+  OLD_JSON: filmlists/Filmliste-akt
 #  JSON_COMPRESSED_XZ: target/filmlists/filmliste.json.xz
 #  OLD_JSON_COMPRESSED_XZ: target/filmlists/filmliste_old.json.xz
 #  JSON_COMPRESSED_GZIP: target/filmlists/filmliste.json.gz
@@ -76,7 +76,7 @@ filmlistSavePaths:
 # If not set no difference film lists will be safed.
 filmlistDiffSavePaths:
 #  JSON: target/filmlists/filmliste_diff.json
-  OLD_JSON: target/filmlists/filmliste_old_diff.json
+  OLD_JSON: filmlists/Filmliste-diff
 #  JSON_COMPRESSED_XZ: target/filmlists/filmliste_diff.json.xz
 #  OLD_JSON_COMPRESSED_XZ: target/filmlists/filmliste_old_diff.json.xz
 #  JSON_COMPRESSED_GZIP: target/filmlists/filmliste_diff.json.gz
@@ -87,24 +87,19 @@ filmlistDiffSavePaths:
 #Sets if a filmlist hash file should be written
 writeFilmlistHashFileEnabled: false
 #The filmlist hash file path
-filmlistHashFilePath: target/filmlists/filmlist.hash
+filmlistHashFilePath: filmlists/filmlist.hash
 
 #Sets if a filmlist id file should be written
 writeFilmlistIdFileEnabled: true
 #The fimlist id file path
-filmlistIdFilePath: target/filmlists/filmlist.id
+filmlistIdFilePath: filmlists/filmlist.id
 
 
 # import additional filmlist sources
 importFilmlistConfigurations :
     - active: true
-      path: "https://verteiler1.mediathekview.de/Filmliste-akt.xz"
-      format: OLD_JSON_COMPRESSED_XZ
-      createDiff: false
-      checkImportListUrl: false
-    - active: true
-      path: "https://c1.mvorg.de/filme-org.xz"
-      format: OLD_JSON_COMPRESSED_XZ
+      path: hist/Filme-org
+      format: OLD_JSON
       createDiff: true
       checkImportListUrl: true
 
@@ -148,7 +143,7 @@ senderConfigurations:
     #2,4,8 ok
     maximumUrlsPerTask: 32
     #10,20,40 ok
-    maximumSubpages: 0
+    maximumSubpages: 40
   ORF:
     #2,4,8 ok
     maximumUrlsPerTask: 40
@@ -171,6 +166,7 @@ senderConfigurations:
     maximumRequestsPerSecond: 8.0
   ZDF:
     maximumRequestsPerSecond: 20.0
+    maximumDaysForSendungVerpasstSection: 21
   FUNK:
     maximumUrlsPerTask: 99
   DW:
@@ -180,6 +176,8 @@ senderConfigurations:
   DREISAT:
     maximumSubpages: 5
     maximumDaysForSendungVerpasstSection: 60
+  PHOENIX:
+    maximumSubpages: 500
 
 # configure string variables
 crawlerApiParams:
@@ -188,20 +186,20 @@ crawlerApiParams:
 #### COPY ####
 copySettings:
   # En- / disables FTP
-  copyEnabled: false
+  copyEnabled: true
 
   # The paths where to safe the film list files.
   # WARNING: You can only set the path for film list formats you listed in "filmlistSaveFormats".
   # Required if enabled
   copyTargetFilePaths:
     # JSON: /var/www/mediathekview/filmlisten/filmliste.json
-    OLD_JSON: /copyTarget/filmliste_old.json
+    OLD_JSON: hist/Filme-org
     # JSON_COMPRESSED: /var/www/mediathekview/filmlisten/filmliste.json.xz
     # OLD_JSON_COMPRESSED_XZ: copyTarget/filmliste_old.json.xz
 
-  copyTargetDiffFilePaths:
+  #copyTargetDiffFilePaths:
     # JSON: /var/www/mediathekview/filmlisten/filmliste_diff.json
-    OLD_JSON: /copyTarget/filmliste_old_diff.json
+    #OLD_JSON: /copyTarget/filmliste_old_diff.json
     # JSON_COMPRESSED: /var/www/mediathekview/filmlisten/filmliste_diff.json.xz
     # OLD_JSON_COMPRESSED_XZ: copyTarget/filmliste_old_diff.json.xz
 
@@ -239,4 +237,4 @@ logSettings:
 
   # The pattern of the file name of the archived log files.
   # See: https://logging.apache.org/log4j/2.0/manual/appenders.html#RollingFileAppender
-  logFileRollingPattern: logs/${date:yyyy-MM}/server-%d{MM-dd-yyyy}-%i.log
+  logFileRollingPattern: logs/${date:yyyy-MM}/server-%d{MM-dd-yyyy}-%i.log
diff --git a/config/Test-MServer-Config-R2.yaml → config/MServer-Config-R2.yaml b/config/Test-MServer-Config-R2.yaml → config/MServer-Config-R2.yaml
@@ -17,22 +17,22 @@ maximumRequestsPerSecond: 999.0
 
 # If set only these Sender will be crawled all other will be ignored.
 senderIncluded:
-  #- ARD
+  - ARD
   #- ARTE_DE
-  #- ARGE_FR
+  #- ARTE_FR
   #- ARTE_EN
   #- ARTE_PL
   #- ARTE_IT
   #- ARTE_ES
-  #- DREISAT
-  #- FUNK
-  #- KIKA
+  - DREISAT
+  - FUNK
+  - KIKA
   - DW
-  #- ORF
-  #- PHOENIX
-  #- SRF
-  #- SR
-  #- ZDF
+  - ORF
+  - PHOENIX
+  - SRF
+  - SR
+  - ZDF
 
 # If set the server will be awake after the crawler run and restarts the run after the given amount.
 #schedules:
@@ -64,7 +64,7 @@ filmlistSaveFormats:
 # The paths where which filmlist should be safed to.
 filmlistSavePaths:
 #  JSON: target/filmlists/filmliste.json
-  OLD_JSON: target/filmlists/filmliste_old.json
+  OLD_JSON: filmlists/Filmliste-akt
 #  JSON_COMPRESSED_XZ: target/filmlists/filmliste.json.xz
 #  OLD_JSON_COMPRESSED_XZ: target/filmlists/filmliste_old.json.xz
 #  JSON_COMPRESSED_GZIP: target/filmlists/filmliste.json.gz
@@ -76,7 +76,7 @@ filmlistSavePaths:
 # If not set no difference film lists will be safed.
 filmlistDiffSavePaths:
 #  JSON: target/filmlists/filmliste_diff.json
-  OLD_JSON: target/filmlists/filmliste_old_diff.json
+  OLD_JSON: filmlists/Filmliste-diff
 #  JSON_COMPRESSED_XZ: target/filmlists/filmliste_diff.json.xz
 #  OLD_JSON_COMPRESSED_XZ: target/filmlists/filmliste_old_diff.json.xz
 #  JSON_COMPRESSED_GZIP: target/filmlists/filmliste_diff.json.gz
@@ -87,24 +87,19 @@ filmlistDiffSavePaths:
 #Sets if a filmlist hash file should be written
 writeFilmlistHashFileEnabled: false
 #The filmlist hash file path
-filmlistHashFilePath: target/filmlists/filmlist.hash
+filmlistHashFilePath: filmlists/filmlist.hash
 
 #Sets if a filmlist id file should be written
 writeFilmlistIdFileEnabled: true
 #The fimlist id file path
-filmlistIdFilePath: target/filmlists/filmlist.id
+filmlistIdFilePath: filmlists/filmlist.id
 
 
 # import additional filmlist sources
 importFilmlistConfigurations :
     - active: true
-      path: "https://verteiler1.mediathekview.de/Filmliste-akt.xz"
-      format: OLD_JSON_COMPRESSED_XZ
-      createDiff: false
-      checkImportListUrl: false
-    - active: true
-      path: "https://c1.mvorg.de/filme-org.xz"
-      format: OLD_JSON_COMPRESSED_XZ
+      path: hist/Filme-org
+      format: OLD_JSON
       createDiff: true
       checkImportListUrl: false
 
@@ -128,7 +123,7 @@ topicsSearchEnabled: false
 # The maximum amount of sub pages to be crawled.<br>
 # Example: If a Sendung overview side has 10 pages with videos for this Sendung and
 # the amount set by this is 5 then the crawler crawls pages 1 to 5.
-maximumSubpages: 1
+maximumSubpages: 5
 
 # The maximum amount of days going to past will be crawled for the "Sendung Verpasst?" section.
 maximumDaysForSendungVerpasstSection: 7
@@ -148,7 +143,7 @@ senderConfigurations:
     #2,4,8 ok
     maximumUrlsPerTask: 32
     #10,20,40 ok
-    maximumSubpages: 0
+    maximumSubpages: 40
   ORF:
     #2,4,8 ok
     maximumUrlsPerTask: 40
@@ -171,37 +166,40 @@ senderConfigurations:
     maximumRequestsPerSecond: 8.0
   ZDF:
     maximumRequestsPerSecond: 20.0
+    maximumDaysForSendungVerpasstSection: 21
   FUNK:
     maximumUrlsPerTask: 99
   DW:
-    maximumSubpages: 1
+    maximumSubpages: 10
   SR:
-    maximumSubpages: 1
+    maximumSubpages: 5
   DREISAT:
-    maximumSubpages: 1
-    maximumDaysForSendungVerpasstSection: 20
-
+    maximumSubpages: 5
+    maximumDaysForSendungVerpasstSection: 60
+  PHOENIX:
+    maximumSubpages: 5
+
 # configure string variables
 crawlerApiParams:
   FUNK_REQUEST_TOKEN: 137782e774d7cadc93dcbffbbde0ce9c
 
 #### COPY ####
 copySettings:
   # En- / disables FTP
-  copyEnabled: true
+  copyEnabled: false
 
   # The paths where to safe the film list files.
   # WARNING: You can only set the path for film list formats you listed in "filmlistSaveFormats".
   # Required if enabled
   copyTargetFilePaths:
     # JSON: /var/www/mediathekview/filmlisten/filmliste.json
-    OLD_JSON: /copyTarget/filmliste_old.json
+    OLD_JSON: hist/Filme-org
     # JSON_COMPRESSED: /var/www/mediathekview/filmlisten/filmliste.json.xz
     # OLD_JSON_COMPRESSED_XZ: copyTarget/filmliste_old.json.xz
 
-  copyTargetDiffFilePaths:
+  #copyTargetDiffFilePaths:
     # JSON: /var/www/mediathekview/filmlisten/filmliste_diff.json
-    OLD_JSON: /copyTarget/filmliste_old_diff.json
+    #OLD_JSON: /copyTarget/filmliste_old_diff.json
     # JSON_COMPRESSED: /var/www/mediathekview/filmlisten/filmliste_diff.json.xz
     # OLD_JSON_COMPRESSED_XZ: copyTarget/filmliste_old_diff.json.xz
 
@@ -216,7 +214,7 @@ logSettings:
   logLevelFile: INFO
 
   # Should be logged to console?
-  logActivateConsole: false
+  logActivateConsole: true
 
   # Should be logged to file?
   logActivateFile: true
@@ -239,4 +237,4 @@ logSettings:
 
   # The pattern of the file name of the archived log files.
   # See: https://logging.apache.org/log4j/2.0/manual/appenders.html#RollingFileAppender
-  logFileRollingPattern: logs/${date:yyyy-MM}/server-%d{MM-dd-yyyy}-%i.log
+  logFileRollingPattern: logs/${date:yyyy-MM}/server-%d{MM-dd-yyyy}-%i.log