Read in ARPA data locally

Makefile

@@ -51,7 +51,7 @@ analysis/output_data/arpa_wtfs.json: ## Pull hand-curated WTF examples from Airt
 		-o $@
 
 src/assets/data/arpa_wtfs.json: analysis/output_data/arpa_wtfs.json ## move wtf data from source folder to graphics data folder
-	cp -R $< $@
+	$(PYENV) python analysis/filter_by_wtf.py $< $@
 
 ##@ Upload/sync
 

analysis/filter_by_wtf.py

@@ -0,0 +1,28 @@
+import pandas as pd
+import click
+import json
+
+@click.command()
+@click.argument('input_filename')
+@click.argument('output_filename')
+def process(input_filename, output_filename):
+    df = read_csv(input_filename)
+    filtered_data = filter_data(df)
+    export(filtered_data, output_filename)
+
+def read_csv(input_filename):
+    f = open(input_filename)
+    data = json.load(f)['data']['requests']
+    df = pd.DataFrame(data)
+    return df
+
+def filter_data(df):
+    filtered_data = df[df['wtf'] == True]
+    return filtered_data
+
+def export(filtered_data, output_filename):
+    filtered_data.to_json(output_filename, orient='records')
+    print("export filter graphics data")
+
+if __name__ == '__main__':
+    process()

src/Randousel/Randousel.svelte

@@ -1,19 +1,18 @@
 <script>
   import { Card, CardSubtitle, CardTitle, CardText, CardActions, Button, MaterialAppMin } from 'svelte-materialify';
   import { format } from 'd3';
+  import wtfDataRaw from '../assets/data/arpa_wtfs.json';
 
   export let location;
-  export let wtfData;
   let theme = 'light';
   let count = 0;
-  let data = []; // Initialize data
+  let data = wtfDataRaw; // Initialize data
   let selectedIndex = 0; // Randomly pick an array element
   let selectedWTF;
   let timeBeforeClick;
 
-  const loading = Promise.all([location, wtfData])
-    .then( ([locationResponse, wtfResponse]) => {
-      data = wtfResponse['data']['requests'].filter(d => d.wtf);
+  const loading = Promise.all([location])
+    .then( ([locationResponse]) => {
       selectedIndex = data.findIndex(d => d.state === locationResponse.region);
     });
 

src/graphic.js

@@ -1,10 +1,8 @@
 import Randousel from './Randousel/Randousel.svelte';
-import wtfDataRaw from './assets/data/arpa_wtfs.json';
 const cacheSlug = 'arpa-categories-geoip-location';
 
 // Provide our components with promises
 const location = getGeoIpData();
-const wtfData = getWTFData();
 
 const randouselElement = document.getElementById('arpa-categories-randousel');
 
@@ -17,7 +15,6 @@ const randouselApp = new Randousel({
   target: randouselElement,
   props: {
     location,
-    wtfData,
   },
 });
 
@@ -42,32 +39,6 @@ function trackVisibility() {
   }
 }
 
-// Read in data from Airtable
-// You can make this with `make graphics_data`
-function getWTFData() {
-  try {
-    return fetch('https://api.baseql.com/airtable/graphql/appjNDCHFduMTVuRA', {
-      headers: {
-        'accept': 'application/json',
-        'accept-language': 'en-US,en;q=0.9',
-        'authorization':
-          'Bearer MTJhNGIzNjgtZmRkZS00MWJiLWIxMjAtYzU4NzVkNDM2YTI5',
-        'cache-control': 'no-cache',
-        'content-type': 'application/json',
-        'pragma': 'no-cache',
-      },
-      referrer: 'https://app.baseql.com/',
-      referrerPolicy: 'strict-origin-when-cross-origin',
-      body: '{"query":"query MyQuery {\\n  requests {\\n    budget\\n    category\\n    description\\n    id\\n    obligations\\n    place\\n    projectName\\n    rank\\n    state\\n    wtf\\n    topPicks\\n  }\\n}\\n","variables":null,"operationName":"MyQuery"}',
-      method: 'POST',
-      mode: 'cors',
-      // "credentials": "include"
-    }).then((r) => r.json());
-  } catch (err) {
-    console.log('fetching error while getting data:', err);
-  }
-}
-
 // Helper to get GeoIP response
 function getGeoIpResponse() {
   try {