Merge pull request #168 from m-lab/sandbox-ptdedup

update traceroute dedup sql

cloud/bq/dedup.go

@@ -177,11 +177,10 @@ var dedupTemplateSwitch = `
 // as mlab-oti:batch.traceroute_20170601.
 var dedupTemplateTraceroute = `
 	#standardSQL
-	# Select single row based on test_id, client_ip, server_ip, src_ip, dest_ip
+	# Select single row based on TestTime, client_ip, server_ip
 	SELECT * EXCEPT (row_number)
     FROM ( SELECT *, ROW_NUMBER() OVER (
-        PARTITION BY CONCAT(test_id, connection_spec.client_ip, connection_spec.server_ip,
-            paris_traceroute_hop.src_ip, paris_traceroute_hop.dest_ip)
+        PARTITION BY CONCAT(STRING(TestTime), Source.IP, DESTINATION.IP)
 		) row_number
 	    FROM ` + "`%s`" + `)
 	WHERE row_number = 1`

cloud/bq/sanity.go

@@ -147,7 +147,8 @@ func (at *AnnotatedTable) CheckIsRegular(ctx context.Context) error {
 }
 
 // GetTableDetail fetches more detailed info about a partition or table.
-// Expects table to have test_id, and task_filename fields.
+// Expects table to have test_id, and task_filename fields for legacy tables,
+// but it is not true for new traceroute tables.
 func GetTableDetail(ctx context.Context, dsExt *dataset.Dataset, table bqiface.Table) (*Detail, error) {
 	// If table is a partition, then we have to separate out the partition part for the query.
 	parts := strings.Split(table.TableID(), "$")
@@ -175,6 +176,13 @@ func GetTableDetail(ctx context.Context, dsExt *dataset.Dataset, table bqiface.T
     FROM `+"`%s.%s`"+`
 		%s  -- where clause`,
 		dataset, tableName, where)
+
+	tracerouteQuery := fmt.Sprintf(`
+		#standardSQL
+		SELECT COUNT(DISTINCT ParseInfo.TaskFileName) AS TestCount, COUNT(DISTINCT ParseInfo.TaskFileName) AS TaskFileCount
+    FROM `+"`%s.%s`"+`
+		%s  -- where clause`,
+		dataset, tableName, where)
 
 	legacyNDTQuery := fmt.Sprintf(`
 		#standardSQL
@@ -190,6 +198,8 @@ func GetTableDetail(ctx context.Context, dsExt *dataset.Dataset, table bqiface.T
 		query = tcpinfoQuery
 	} else if parts[0] == "legacy" {
 		query = legacyNDTQuery
+	} else if parts[0] == "traceroute" {
+		query = tracerouteQuery
 	}
 	err := dsExt.QueryAndParse(ctx, query, &detail)
 	if err != nil {