-
Notifications
You must be signed in to change notification settings - Fork 2
/
rf.r
3046 lines (2544 loc) · 113 KB
/
rf.r
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
tos all
#try random forest
library(randomForest);
pairs <- readRDS("../MODELS_PHASE5/match2345_pairs.RDS");
pairs$pairs$im = as.factor(pairs$pairs$"is_match");
mm = pairs$pairs$"is_match" == 1;
nmm=!mm & apply(pairs$pairs[,c("n","e","ln","fn","un","ifn","ln1","fn1")], 1, max) > .8 | pairs$pairs$ad>0|pairs$pairs$tdz>.9|pairs$pairs$d2vSim>0;
nnmm=!mm & !nmm
vv = 1:dim(pairs$pairs)[1];
#a=paste(pairs$pairs$id1,pairs$pairs$id2,sep=";");
#b=paste(pairs$pairs$id2,pairs$pairs$id1,sep=";");
#ab=match(a,b);
#sum(pairs$pairs$im!=pairs$pairs$im[ab])
#Fit
rf = list();
p = list();
sel=sample(0:9,sum(mm),replace=T);
sel1=sample(0:9,sum(nmm),replace=T);
sel2=sample(0:9,sum(nnmm),replace=T);
for (i in 0:9){
mm1t = vv[mm][sel!=i];
mm1v = vv[mm][sel==i];
mm0t = vv[nmm][sel1!=i];
mm0v = vv[nmm][sel1==i];
mm0v = c(mm0v, vv[nnmm][sel2==i])
dt=pairs$pairs[c(mm1t,mm0t),c("n","e","ln","fn","un","d2vSim", "ad", "tdz", "ifn", "ln1f", "fnf", "ln1","fn1", "im")];
dv=pairs$pairs[c(mm1v,mm0v),c("n","e","ln","fn","un","d2vSim","ad", "tdz", "ifn", "ln1f", "fnf", "ln1","fn1","im")];
rf[[i+1]] = randomForest(im ~ ., dt, importance=T);
p[[i+1]] = predict(rf[[i+1]],dv);
print (table(p[[i+1]],dv$im));
}
0 1
0 549693 1
1 11 1028
0 1
0 548507 1
1 16 999
0 1
0 549742 0
1 20 989
0 1
0 549492 3
1 14 1046
0 1
0 549225 3
1 15 1010
0 1
0 549840 0
1 20 1016
0 1
0 550595 1
1 12 1086
0 1
0 548972 3
1 15 1060
0 1
0 548861 2
1 13 1032
#inspect errors
res = c();
for (i in 0:9){
mm1v = vv[mm][sel==i];
mm0v = vv[nmm][sel1==i];
mm0v = c(mm0v, vv[nnmm][sel2==i])
dv=pairs$pairs[c(mm1v,mm0v),c("n","e","ln","fn","un","d2vSim","ad", "tdz", "ifn", "ln1f", "fnf", "ln1","fn1","im","id1","id2")];
miss = dv[dv$im!=p[[i+1]],c("id1","id2","im")];
ss = match(paste(pairs$pairs$id1, pairs$pairs$id2,sep=";"),paste(miss$id1,miss$id2,sep=";"),nomatch=0)>0;
misd = pairs$pairs[ss,];
aa = paste(pairs$data1[misd$id1,"a"],pairs$data2[misd$id2,"a"], sep=";");
res = rbind(res, cbind(misd[,c("id1","id2","d2vSim","ad", "tdz","im")], aa));
}
#save
save(rf, p, res, sel, sel1, sel2, file = "/home/audris/rfmodels.RData");
res[res$im==0,c("id1","id2","d2vSim","ad","tdz","im")];
id1 id2 d2vSim ad tdz im
1817022 775 1992 0.0000000 0.0000000000 0.992805755 0
2620775 1118 1410 0.0000000 0.0000000000 0.782660190 0
3198135 1364 1900 0.0000000 0.0000000000 -0.007196270 0
3823436 1631 1086 0.0000000 0.0000000000 -0.011670606 0
4923380 2100 1225 0.0000000 0.0447984072 0.768176577 0
2469101 1053 2161 0.5482726 0.0000000000 0.094913538 0
5117770 2183 980 0.7300389 0.0000000000 0.992805755 0
429508 184 373 0.0000000 2.9642246777 0.985368371 0
1674130 714 2145 0.0000000 0.0000000000 -0.007170286 0
2018882 861 2182 0.0000000 0.0000000000 0.341861193 0
2469100 1053 2160 0.7403789 6.3988278451 0.247812187 0
2863731 1222 486 0.0000000 0.0000000000 -0.007194255 0
4667324 1991 774 0.0000000 0.0000000000 0.630051985 0
5063908 2160 1053 0.7403789 6.3988278451 0.247812187 0
5351827 2283 537 0.0000000 0.0000000000 -0.007183737 0
1674131 714 2146 0.0000000 0.0000000000 -0.007170286 0
2566524 1095 1094 0.0000000 0.0000000000 0.548521202 0
3924916 1674 1731 0.0000000 0.0000000000 -0.007194245 0
4669669 1992 774 0.0000000 0.0000000000 0.992805755 0
5028394 2145 714 0.0000000 0.0000000000 -0.007170286 0
5066253 2161 1053 0.5482726 0.5440382033 0.094913538 0
1814677 774 1992 0.0000000 0.0000000000 0.992805755 0
3198134 1364 1899 0.0000000 0.0000000000 -0.006953688 0
3305223 1410 1118 0.0000000 0.0000000000 0.782660190 0
4454519 1900 1364 0.0000000 0.0000000000 -0.007196270 0
4603843 1964 608 0.0000000 0.0000000000 0.391344878 0
3647564 1556 1089 0.0000000 0.0000000000 0.081967324 0
3943395 1682 1450 0.0000000 0.0217592593 0.992805755 0
4452175 1899 1365 0.0000000 0.0005681818 -0.007611807 0
4667325 1991 775 0.0000000 0.0000000000 0.630051985 0
1425379 608 1964 0.0000000 0.0000000000 0.391344878 0
2552916 1089 1556 0.0000000 0.0000000000 0.081967324 0
4452174 1899 1364 0.0000000 0.0000000000 -0.006953688 0
5026049 2144 714 0.0000000 0.0000000000 -0.007170286 0
5030739 2146 714 0.0000000 0.0000000000 -0.007170286 0
5115306 2182 861 0.0000000 0.0000000000 0.341861193 0
1814676 774 1991 0.0000000 0.0000000000 0.630051985 0
2021226 862 2181 0.0000000 0.0000000000 -0.007801478 0
2550570 1088 1555 0.0000000 0.0000000000 0.783570825 0
#manually correct errors
eq= c(1817022,2620775,3198135,3823436,4923380,2469101,5117770,429508 ,1674130,2018882,2469100,
2863731,4667324,5063908,5351827,1674131 ,2566524,3924916,4669669,5028394,5066253,
1814677,3198134,3305223,4454519,4603843,3647564,3943395,4452175,4667325,1425379,2552916,
4452174,5026049,5030739,5115306,1814676,2021226,2550570);
pairs$pairs$im=as.factor(pairs$pairs$"is_match");
fr=c(1117,1771,2183,1632,2265,2265,1221, 666,1900,487,1053,1075,1085,1085,1207,1411,748, 1411, 1221, 1632, 1076, 1148, 748, 774, 775, 774, 1115, 1053, 981, 185, 1093, 1899, 2099, 373, 966, 1094, 184, 2100, 2144, 424, 425, 1450, 1228, 1116, 775, 1117, 1364, 774, 1086, 1900, 1118, 1227, 1227);
to=c(1411, 666, 980,1085, 749, 748,487,1772,1365,1221,2160,1207,1632,1631,1076,1117,2264,1118, 486, 1086, 1207, 528, 2264, 1992, 1992, 775, 1409, 2161, 2183, 373, 1096, 1365, 1225, 184, 1619,1096, 373, 1225, 2255, 229, 230, 1683, 2099, 1410, 1991, 1410, 1899, 1991, 1631, 1364, 1410, 2100, 2099);
for (i in c(1:length(fr))){
pairs$pairs$im[pairs$pairs$id1==fr[i]&pairs$pairs$id2==to[i]]=1;
pairs$pairs$im[pairs$pairs$id2==fr[i]&pairs$pairs$id1==to[i]]=1;
}
pp=pairs$pairs[eq,]
pp=pp[pp$im==0,]
paste(pairs$data1[pp$id1,"a"],pairs$data1[pp$id2,"a"])
[1] "yong sheng gong <[email protected]> yong sheng gong <[email protected]>"
[2] "hayderimran7 <[email protected]> imran <[email protected]>"
[3] "abhiram_moturi <[email protected]> abhiram moturi <[email protected]>"
[4] "Elizabeth Elwell <[email protected]> Beth Elwell <[email protected]>"
[5] "yong sheng gong <[email protected]> Yong Sheng Gong <[email protected]>"
[6] "fabien Boucher <[email protected]> Fabien Boucher <[email protected]>"
[7] "Zara <[email protected]> Zahra <[email protected]>"
[8] "yong sheng gong <[email protected]> yong sheng gong <[email protected]>"
[9] "Joe Talerico <[email protected]> Joe <[email protected]>"
[10] "K Jonathan Harker <[email protected]> Jonathan Harker <[email protected]>"
[11] "Anju Tiwari <[email protected]> Anju Tiwari <[email protected]>"
[12] "Joe <[email protected]> Joe Talerico <[email protected]>"
[13] "Jonathan Harker <[email protected]> K Jonathan Harker <[email protected]>"
[14] "Yong Sheng Gong <[email protected]> yong sheng gong <[email protected]>"
[15] "Yong Sheng Gong <[email protected]> yong sheng gong <[email protected]>"
[16] "imran <[email protected]> hayderimran7 <[email protected]>"
[17] "hayderimran7 <[email protected]> imran <[email protected]>"
[18] "Jonathan Harker <[email protected]> K Jonathan Harker <[email protected]>"
for (i in c(1:length(pp$id1))){
pairs$pairs$im[pairs$pairs$id1==pp$id1[i]&pairs$pairs$id2==pp$id2[i]]=1;
pairs$pairs$im[pairs$pairs$id2==pp$id1[i]&pairs$pairs$id1==pp$id2[i]]=1;
}
pp=pairs$pairs[pairs$pairs$im!=pairs$pairs$"is_match",];
pp=pp[pp$im==1&pp$id1<pp$id2,];
paste(pairs$data1[pp$id1,"a"],pairs$data1[pp$id2,"a"]);
[1] "Victoria Martinez de la Cruz <[email protected]> Victoria Martinez de la Cruz <[email protected]>"
[2] "Victoria Mart\303\255nez de la Cruz <[email protected]> Victoria Martinez de la Cruz <[email protected]>"
[3] "ghanshyam <[email protected]> Ghanshyam Mann <[email protected]>"
[4] "ghanshyam <[email protected]> Ghanshyam Mann <[email protected]>"
[5] "abhiram moturi <[email protected]> amoturi <[email protected]>"
[6] "abhiram moturi <[email protected]> abhiram_moturi <[email protected]>"
[7] "Abhiram Moturi <[email protected]> amoturi <[email protected]>"
[8] "Sergey Vasilenko <[email protected]> vsaienko <[email protected]>"
[9] "Beth Elwell <[email protected]> Elizabeth Elwell <[email protected]>"
[10] "Joe <[email protected]> Joe Talerico <[email protected]>"
[11] "Sukhdev Kapur <[email protected]> sukhdev <[email protected]>"
[12] "Sukhdev Kapur <[email protected]> Sukhdev <[email protected]>"
[13] "yong sheng gong <[email protected]> Yong Sheng Gong <[email protected]>"
[14] "yong sheng gong <[email protected]> yong sheng gong <[email protected]>"
[15] "yong sheng gong <[email protected]> Yong Sheng Gong <[email protected]>"
[16] "Lingxian Kong <[email protected]> lingxiankong <[email protected]>"
[17] "Lingxian Kong <[email protected]> LingxianKong <[email protected]>"
[18] "Lingxian Kong <[email protected]> LingxianKong <[email protected]>"
[19] "pcarlton <[email protected]> pcarlton <[email protected]>"
[20] "pcarlton <[email protected]> paul-carlton2 <[email protected]>"
[21] "paul-carlton2 <[email protected]> pcarlton <[email protected]>"
[22] "paul-carlton2 <[email protected]> paul-carlton2 <[email protected]>"
[23] "hayderimran7 <[email protected]> imran <[email protected]>"
[24] "hayderimran7 <[email protected]> imran <[email protected]>"
[25] "watanabe.fumitaka <[email protected]> watanabe.fumitaka <[email protected]>"
[26] "David Purcell <[email protected]> David Purcell <[email protected]>"
[27] "David Purcell <[email protected]> David Purcell <[email protected]>"
[28] "Salvatore Orlando <[email protected]> Salvatore <[email protected]>"
[29] "Salvatore Orlando <[email protected]> salvatore <[email protected]>"
[30] "janki <[email protected]> Janki Chhatbar <[email protected]>"
[31] "Janki <[email protected]> Janki Chhatbar <[email protected]>"
[32] "Ulrik Sjolin <[email protected]> Ulrik Sjolin <[email protected]>"
[33] "Ulrik Sjolin <[email protected]> Ulrik Sj\303\266lin <[email protected]>"
[34] "Ulrik Sj\303\266lin <[email protected]> Ulrik Sjolin <[email protected]>"
[35] "Ulrik Sj\303\266lin <[email protected]> Ulrik Sj\303\266lin <[email protected]>"
[36] "Jonathan Harker <[email protected]> K Jonathan Harker <[email protected]>"
[37] "Jonathan Harker <[email protected]> K Jonathan Harker <[email protected]>"
[38] "Fabien Boucher <[email protected]> Fabien Boucher <[email protected]>"
[39] "Fabien Boucher <[email protected]> fabien Boucher <[email protected]>"
[40] "Fabien Boucher <[email protected]> Fabien Boucher <[email protected]>"
[41] "Emilien Macchi <[email protected]> emilienm <[email protected]>"
[42] "Emilien Macchi <[email protected]> EmilienM <[email protected]>"
[43] "Emilien Macchi <[email protected]> EmilienM <[email protected]>"
[44] "Emilien Macchi <[email protected]> emilienm <[email protected]>"
[45] "\303\211milien Macchi <[email protected]> EmilienM <[email protected]>"
[46] "\303\211milien Macchi <[email protected]> emilienm <[email protected]>"
[47] "Kevin Benton <[email protected]> Mr. Bojangles <[email protected]>"
[48] "Kevin Benton <[email protected]> Mr. Bojangles <[email protected]>"
[49] "Kevin Benton <[email protected]> Mr. Bojangles <[email protected]>"
[50] "Kevin Benton <[email protected]> Mr. Bojangles <[email protected]>"
[51] "Kevin Benton <[email protected]> Mr. Bojangles <[email protected]>"
[52] "irina povolotskaya <[email protected]> Irina <[email protected]>"
[53] "irina povolotskaya <[email protected]> ipovolotskaya <[email protected]>"
[54] "Irina Povolotskaya <[email protected]> Irina <[email protected]>"
[55] "Irina Povolotskaya <[email protected]> ipovolotskaya <[email protected]>"
[56] "Anju Tiwari <[email protected]> Anju Tiwari <[email protected]>"
[57] "Anju Tiwari <[email protected]> anju tiwari <[email protected]>"
[58] "Zara <[email protected]> Zahra <[email protected]>"
[59] "Yong Sheng Gong <[email protected]> gongysh <[email protected]>"
#Fit again after correcting errors
mmC = pairs$pairs$im == 1;
nmmC=!mmC & apply(pairs$pairs[,c("n","e","ln","fn","un","ifn","ln1","fn1")], 1, max) > .8 | pairs$pairs$ad>0|pairs$pairs$tdz>.9|pairs$pairs$d2vSim>0;
nnmmC=!mmC & !nmmC
selC=sample(0:9,sum(mmC),replace=T);
selC1=sample(0:9,sum(nmmC),replace=T);
selC2=sample(0:9,sum(nnmmC),replace=T);
rfC = list();
pC = list();
for (i in 0:9){
mm1t = vv[mmC][selC!=i];
mm1v = vv[mmC][selC==i];
mm0t = vv[nmmC][selC1!=i];
#mm0t = c(mm0t, vv[nnmmC][selC2!=i]) #mm0t = c(mm0t, vv[nnmmC&fullP!=pairs$pairs$im])
mm0v = vv[nmmC][selC1==i];
mm0v = c(mm0v, vv[nnmmC][selC2==i])#include the entire validation set for prediction
dt=pairs$pairs[c(mm1t,mm0t),c("n","e","ln","fn","un","d2vSim", "ad", "tdz", "ifn", "ln1f", "fnf", "ln1","fn1", "im")];
dv=pairs$pairs[c(mm1v,mm0v),c("n","e","ln","fn","un","d2vSim","ad", "tdz", "ifn", "ln1f", "fnf", "ln1","fn1","im")];
rfC[[i+1]] = randomForest(im ~ ., dt, importance=T);
pC[[i+1]] = predict(rfC[[i+1]],dv);
print (table(pC[[i+1]],dv$im));
}
0 1
0 549609 3
1 0 993
0 1
0 548181 3
1 2 1108
0 1
0 549470 4
1 0 1079
0 1
0 551139 2
1 0 1039
0 1
0 550112 5
1 1 1012
0 1
0 549211 1
1 3 1067
0 1
0 549404 1
1 0 1032
0 1
0 547961 3
1 0 1019
0 1
0 548730 2
1 0 1086
0 1
0 549569 2
1 0 1010
#very nice results!
resC = c();
for (i in 0:9){
mm1v = vv[mmC][selC==i];
mm0v = vv[nmmC][selC1==i];
mm0v = c(mm0v, vv[nnmmC][selC2==i])#include the entire validation set for prediction
dv=pairs$pairs[c(mm1v,mm0v),c("n","e","ln","fn","un","d2vSim","ad", "tdz", "ifn", "ln1f", "fnf", "ln1","fn1","im","id1","id2")];
miss = dv[dv$im!=pC[[i+1]],c("id1","id2","im")];
ss = match(paste(pairs$pairs$id1, pairs$pairs$id2,sep=";"),paste(miss$id1,miss$id2,sep=";"),nomatch=0)>0;
misd = pairs$pairs[ss,];
aa = paste(pairs$data1[misd$id1,"a"],pairs$data2[misd$id2,"a"], sep=";");
resC = rbind(resC, cbind(misd[,c("id1","id2","d2vSim","ad", "tdz","im")], aa));
}
#save
save(rfC, pC, resC, selC, selC1, selC2, file = "/home/audris/rfmodelsC.RData");
load(file = "/home/audris/rfmodelsC.RData");
resC[resC$im==0,];
id1 id2 d2vSim ad tdz im
128525 55 1895 0 5.690182 0.9928058 0
4441485 1895 55 0 5.690182 0.9928058 0
1202126 513 1486 0 0.000000 0.9840866 0
3401933 1451 1683 0 0.000000 0.9928058 0
3482838 1486 513 0 0.000000 0.9840866 0
3945741 1683 1451 0 0.000000 0.9928058 0
aa
128525 alevine <[email protected]>;Alexandre Levine <[email protected]>
4441485 Alexandre Levine <[email protected]>;alevine <[email protected]>
1202126 Newptone <[email protected]>;Newptone <[email protected]>
3401933 anju Tiwari <[email protected]>;anju tiwari <[email protected]>
3482838 Newptone <[email protected]>;Newptone <[email protected]>
3945741 anju tiwari <[email protected]>;anju Tiwari <[email protected]>
resC[resC$im==1,]
id1 id2 d2vSim ad tdz im
4286144 1828 1829 0 0.00000000 0.992805755 1
4511360 1924 1925 0 0.00000000 0.511877231 1
4513704 1925 1924 0 0.00000000 0.511877231 1
145454 63 64 0 0.00000000 0.265827278 1
4316638 1841 1838 0 0.08396192 -0.007194245 1
37540 17 20 0 0.00000000 -0.009652092 1
1916682 818 817 0 0.00000000 0.992805755 1
3781757 1613 1617 0 0.00000000 0.992783249 1
5088476 2170 2171 0 0.00000000 0.992805755 1
147798 64 63 0 0.00000000 0.265827278 1
2177091 929 931 0 0.00000000 -0.009083949 1
1327838 567 568 0 0.00000000 0.494679582 1
2181779 931 929 0 0.00000000 -0.009083949 1
4288487 1829 1827 0 0.00000000 0.093287908 1
50884761 2170 2171 0 0.00000000 0.992805755 1
5090820 2171 2170 0 0.00000000 0.992805755 1
1330182 568 567 0 0.00000000 0.494679582 1
50908201 2171 2170 0 0.00000000 0.992805755 1
1914338 817 818 0 0.00000000 0.992805755 1
3791133 1617 1613 0 0.00000000 0.992783249 1
42861441 1828 1829 0 0.00000000 0.992805755 1
37817571 1613 1617 0 0.00000000 0.992783249 1
37911331 1617 1613 0 0.00000000 0.992783249 1
19143381 817 818 0 0.00000000 0.992805755 1
19166821 818 817 0 0.00000000 0.992805755 1
aa
4286144 Hua ZHANG <[email protected]>;Zhang Hua <[email protected]>
4511360 Matthieu Huin <[email protected]>;Marius Cornea <[email protected]>
4513704 Marius Cornea <[email protected]>;Matthieu Huin <[email protected]>
145454 root <[email protected]>;root <[email protected]>
4316638 Reedip Banerjee <[email protected]>;Reedip <[email protected]>
37540 Eric Peterson <[email protected]>;eric <[email protected]>
1916682 Masayuki Nakamura <[email protected]>;Jin Hase <[email protected]>
3781757 lawrancejing <[email protected]>;jing.liuqing <[email protected]>
5088476 root <[email protected]>;root <[email protected]>
147798 root <[email protected]>;root <[email protected]>
2177091 dimtruck <[email protected]>;Dimitry Ushakov <[email protected]>
1327838 andrewbogott <[email protected]>;Andrew Bogott <[email protected]>
2181779 Dimitry Ushakov <[email protected]>;dimtruck <[email protected]>
4288487 Zhang Hua <[email protected]>;ZHANG Hua <[email protected]>
50884761 root <[email protected]>;root <[email protected]>
5090820 root <[email protected]>;root <[email protected]>
1330182 Andrew Bogott <[email protected]>;andrewbogott <[email protected]>
50908201 root <[email protected]>;root <[email protected]>
1914338 Jin Hase <[email protected]>;Masayuki Nakamura <[email protected]>
3791133 jing.liuqing <[email protected]>;lawrancejing <[email protected]>
42861441 Hua ZHANG <[email protected]>;Zhang Hua <[email protected]>
37817571 lawrancejing <[email protected]>;jing.liuqing <[email protected]>
37911331 jing.liuqing <[email protected]>;lawrancejing <[email protected]>
19143381 Jin Hase <[email protected]>;Masayuki Nakamura <[email protected]>
19166821 Masayuki Nakamura <[email protected]>;Jin Hase <[email protected]>
#See if the method works with the predictors used in bagging
"n" "e" "ln1" "fn1" "un" "ifn" "ln1f" "fn1f" "ad" "tdz" "d2vSim"
rfC1 = list();
pC1 = list();
for (i in 0:9){
mm1t = vv[mmC][selC!=i];
mm1v = vv[mmC][selC==i];
mm0t = vv[nmmC][selC1!=i];
#mm0t = c(mm0t, vv[nnmmC][selC2!=i]) #mm0t = c(mm0t, vv[nnmmC&fullP!=pairs$pairs$im])
mm0v = vv[nmmC][selC1==i];
mm0v = c(mm0v, vv[nnmmC][selC2==i])#include the entire validation set for prediction
dt=pairs$pairs[c(mm1t,mm0t),c("n","e","ln1","fn1","un","d2vSim", "ad", "tdz", "ifn", "ln1f", "fn1f", "im")];
dv=pairs$pairs[c(mm1v,mm0v),c("n","e","ln1","fn1","un","d2vSim", "ad", "tdz", "ifn", "ln1f", "fn1f", "im")];
rfC1[[i+1]] = randomForest(im ~ ., dt, importance=T);
pC1[[i+1]] = predict(rfC1[[i+1]],dv);
print (table(pC1[[i+1]],dv$im));
}
0 1
0 549609 4
1 0 992
0 1
0 548181 3
1 2 1108
0 1
0 549470 3
1 0 1080
0 1
0 551139 5
1 0 1036
0 1
0 550111 5
1 2 1012
0 1
0 549211 1
1 3 1067
0 1
0 549403 1
1 1 1032
0 1
0 547961 5
1 0 1017
0 1
0 548730 5
1 0 1083
0 1
0 549569 2
1 0 1010
save(rfC1, pC1, file = "/home/audris/rfmodelsC1.RData");
##############################################################################################################
#load ..
#do errors under transitive closure:
#Below is final
n=dim(pairs$data1)[1];
pC1t = pC1
for (i in 0:9){
mm1v = vv[mmC][selC==i];
mm0v = vv[nmmC][selC1==i];
mm0v = c(mm0v, vv[nnmmC][selC2==i])#include the entire validation set for prediction
dv=pairs$pairs[c(mm1v,mm0v),c("id1","id2","n","e","ln1","fn1","un","d2vSim", "ad", "tdz", "ifn", "ln1f", "fn1f", "im")];
crrf = closeG (n, dv[pC1[[i+1]]==1,c("id1","id2")]);
idm = paste(dv[,"id1"],dv[,"id2"],sep=";")
pC1t[[i+1]][match(idm,paste(crrf[,1],crrf[,2],sep=";"),nomatch=0)>0] = 1;
crrf = closeG (n, dv[dv$im==1,c("id1","id2")]);
imt = dv$im;
idm = paste(dv[,"id1"],dv[,"id2"],sep=";")
imt[match(idm,paste(crrf[,1],crrf[,2],sep=";"),nomatch=0)>0] = 1;
print(table(pC1t[[i+1]],imt))
}
0 1
0 549609 4
1 0 992
imt
0 1
0 548179 3
1 2 1110
imt
0 1
0 549469 2
1 0 1082
imt
0 1
0 551136 5
1 0 1039
imt
0 1
0 550108 5
1 3 1014
imt
0 1
0 549204 1
1 2 1075
imt
0 1
0 549402 1
1 1 1033
imt
0 1
0 547958 4
1 0 1021
imt
0 1
0 548730 4
1 0 1084
imt
0 1
0 549569 2
1 0 1010
##############################################################################################################
# a bit worse than above, perhaps add ln?
# produce all 10 full preditions
#pairsf <- readRDS("../MODELS_PHASE4/RDSFiles/full_pairs.RDS");
seq=1:25622405
fullP7 = c()
for (i in 0:9){
fullP7 = c(fullP7, predict(rfC1[[7]],pairsf$pairs[seq+i*25622405,]));
}
fullP7=fullP7[-256224050];
seq=1:25622405
fullP1 = c()
for (i in 0:9){
fullP1 = c(fullP1, predict(rfC1[[1]],pairsf$pairs[seq+i*25622405,]));
}
fullP1=fullP1[-256224050];
n=dim(pairsf$data1)[1];
calcMeas(n,pairsf$pairs[fullP1==2,1:2]);
n nid nid+/nid n+ n1
1.60070e+04 1.07950e+04 2.71144e-01 8.13900e+03 7.86800e+03
seq=1:25622405
fullP2 = c()
for (i in 0:9){
fullP2 = c(fullP2, predict(rfC1[[2]],pairsf$pairs[seq+i*25622405,]));
}
fullP2=fullP2[-256224050];
calcMeas(n,pairsf$pairs[fullP2==2,1:2]);
n nid nid+/nid n+ n1
1.60070e+04 1.07900e+04 2.70899e-01 8.14000e+03 7.86700e+03
fullP3 = c()
for (i in 0:9){
fullP3 = c(fullP3, predict(rfC1[[3]],pairsf$pairs[seq+i*25622405,]));
}
fullP3=fullP3[-256224050];
calcMeas(n,pairsf$pairs[fullP3==2,1:2]);
n nid nid+/nid n+ n1
1.600700e+04 1.076300e+04 2.707424e-01 8.158000e+03 7.849000e+03
fullP4 = c()
for (i in 0:9){
fullP4 = c(fullP4, predict(rfC1[[4]],pairsf$pairs[seq+i*25622405,]));
}
fullP4=fullP4[-256224050];
calcMeas(n,pairsf$pairs[fullP4==2,1:2]);
n nid nid+/nid n+ n1
1.600700e+04 1.080800e+04 2.699852e-01 8.117000e+03 7.890000e+03
fullP5 = c()
for (i in 0:9){
fullP5 = c(fullP5, predict(rfC1[[5]],pairsf$pairs[seq+i*25622405,]));
}
fullP5=fullP5[-256224050];
fullP6 = c()
for (i in 0:9){
fullP6 = c(fullP6, predict(rfC1[[6]],pairsf$pairs[seq+i*25622405,]));
}
fullP6=fullP6[-256224050];
fullP8 = c()
for (i in 0:9){
fullP8 = c(fullP8, predict(rfC1[[8]],pairsf$pairs[seq+i*25622405,]));
}
fullP8=fullP8[-256224050];
fullP7ullP9 = c()
for (i in 0:9){
fullP9 = c(fullP9, predict(rfC1[[9]],pairsf$pairs[seq+i*25622405,]));
}
fullP9=fullP9[-256224050];
fullP10 = c()
for (i in 0:9){
fullP10 = c(fullP10, predict(rfC1[[10]],pairsf$pairs[seq+i*25622405,]));
}
fullP10=fullP10[-256224050];
save(fullP1, fullP2, fullP3, fullP4, fullP5, fullP6, fullP7, fullP8, fullP9, fullP10, file = "/home/audris/rfmodelsC1FullP.RData");
################################
#do further refinement with Yuxia
################################
load("/home/audris/rfmodelsC1FullP.RData");
fullP = cbind(fullP1,fullP2,fullP3,fullP4,fullP5,fullP6,fullP7,fullP8,fullP9,fullP1)
agg = apply(fullP,1,sum);
table(agg);
10 11 12 13 14 15 16 17
256192190 226 217 116 93 71 103 121
18 19 20
151 186 30575
sel = agg < 20 & agg > 10
mismatch=data.frame(fr=pairsf$data1$a[pairsf$pairs$id1[sel]],to=pairsf$data1$a[pairsf$pairs$id2[sel]])
mismatch=data.frame(fr=pairsf$data1$a[pairsf$pairs$id1[sel]],to=pairsf$data1$a[pairsf$pairs$id2[sel]],ad=pairsf$pairs$ad[sel],d2vSim=pairsf$pairs$d2vSim[sel],tdz=pairsf$pairs$tdz[sel])
write.table(mismatch,"/home/audris/mismatch.csv",sep=";",quote=F)
################################
#try simpler models that can be applied on the entire universe
rfCS = list();
pCS = list();
for (i in 0:9){
mm1t = vv[mmC][selC!=i];
mm1v = vv[mmC][selC==i];
mm0t = vv[nmmC][selC1!=i];
#mm0t = c(mm0t, vv[nnmmC][selC2!=i]) #mm0t = c(mm0t, vv[nnmmC&fullP!=pairs$pairs$im])
mm0v = vv[nmmC][selC1==i];
mm0v = c(mm0v, vv[nnmmC][selC2==i])#include the entire validation set for prediction
dt=pairs$pairs[c(mm1t,mm0t),c("n","e","ln","fn","un", "ifn", "d2vSim", "im")];
dv=pairs$pairs[c(mm1v,mm0v),c("n","e","ln","fn","un", "ifn", "d2vSim", "im")];
rfCS[[i+1]] = randomForest(im ~ ., dt, importance=T);
pCS[[i+1]] = predict(rfCS[[i+1]],dv);
print (table(pCS[[i+1]],dv$im));
}
0 1
0 549602 3
1 7 993
0 1
0 548174 6
1 9 1105
0 1
0 549463 2
1 7 1081
0 1
0 551137 3
1 2 1038
0 1
0 550109 3
1 4 1014
0 1
0 549207 3
1 7 1065
0 1
0 549397 0
1 7 1033
0 1
0 547954 5
1 7 1017
0 1
0 548727 2
1 3 1086
0 1
0 549560 0
1 9 1012
resCS = c();
for (i in 0:9){
mm1v = vv[mmC][selC==i];
mm0v = vv[nmmC][selC1==i];
mm0v = c(mm0v, vv[nnmmC][selC2==i])#include the entire validation set for prediction
dv=pairs$pairs[c(mm1v,mm0v),c("n","e","ln","fn","un","d2vSim","ad", "tdz", "ifn", "ln1f", "fnf", "ln1","fn1","im","id1","id2")];
miss = dv[dv$im!=pCS[[i+1]],c("id1","id2","im")];
ss = match(paste(pairs$pairs$id1, pairs$pairs$id2,sep=";"),paste(miss$id1,miss$id2,sep=";"),nomatch=0)>0;
misd = pairs$pairs[ss,];
aa = paste(pairs$data1[misd$id1,"a"],pairs$data2[misd$id2,"a"], sep=";");
resCS = rbind(resCS, cbind(misd[,c("id1","id2","d2vSim","im")], aa));
}
save(rfCS, pCS, resCS, pairs, file = "/home/audris/rfmodelsCS.RData");
#try even simpler model without the fingerprints
rfCSB = list();
pCSB = list();
for (i in 0:9){
mm1t = vv[mmC][selC!=i];
mm1v = vv[mmC][selC==i];
mm0t = vv[nmmC][selC1!=i];
#mm0t = c(mm0t, vv[nnmmC][selC2!=i]) #mm0t = c(mm0t, vv[nnmmC&fullP!=pairs$pairs$im])
mm0v = vv[nmmC][selC1==i];
mm0v = c(mm0v, vv[nnmmC][selC2==i])#include the entire validation set for prediction
dt=pairs$pairs[c(mm1t,mm0t),c("n","e","ln","fn","un", "ifn", "im")];
dv=pairs$pairs[c(mm1v,mm0v),c("n","e","ln","fn","un", "ifn", "im")];
rfCSB[[i+1]] = randomForest(im ~ ., dt, importance=T);
pCSB[[i+1]] = predict(rfCSB[[i+1]],dv);
print (table(pCSB[[i+1]],dv$im));
}
0 1
0 549601 6
1 8 990
0 1
0 548174 8
1 9 1103
0 1
0 549462 4
1 8 1079
0 1
0 551137 4
1 2 1037
0 1
0 550109 4
1 4 1013
0 1
0 549207 6
1 7 1062
0 1
0 549397 0
1 7 1033
0 1
0 547953 6
1 8 1016
0 1
0 548727 5
1 3 1083
0 1
0 549560 1
1 9 1011
#Now disconnect large clusters
#disconnect:
#first create the graph
load(file = "/home/audris/rfmodelsFullP7.RData");
table(fullP7);
1 2
256193049 31000
#number of clusters
n=dim(pairsf$data1)[1];
gg = make_empty_graph(n, directed = FALSE) # empty graph
gg = set_vertex_attr(g, "id", value = 1:n)
gg = add_edges(gg, as.vector(t(pairsf$pairs[fullP7==2,1:2])));
gg = simplify(gg,remove.multiple = TRUE, remove.loops = TRUE)
clustg7c <- components(gg, "weak") # get clusters
sort(-table(blocksg7$block))[1:10];
10201 1689 2265 1512 549 3405 650 5575 3890 3084
-44 -30 -27 -23 -19 -17 -16 -16 -15 -14
length(table(blocksg7$block))
[1] 10835
n=dim(pairsf$data1)[1];
idmatch = paste(pairsf$pairs$id1,pairsf$pairs$id2,sep=";");
# Break root cluster (based on earlier analysis)
fullP7[pairsf$pairs$id1==14783&fullP7==2&pairsf$pairs$id2!=9138&pairsf$pairs$id2!=14783]=1
fullP7[pairsf$pairs$id2==14783&fullP7==2&pairsf$pairs$id1!=9138&pairsf$pairs$id1!=14783]=1
# Break next cluster 3705:
cl3705=pairsf$pairs[match(pairsf$pairs$id1,blocksg7[blocksg7$block==3705,"id"],nomatch=0)>0&fullP7==2,c("id1","id2")]
cl3705=cl3705[cl3705$id1<cl3705$id2,];
all=paste(cl3705$id1,cl3705$id2,sep=";");
all=all[-c(37,39,28, 26,25,23,19,17,13,11,1,2)];
all1=paste(cl3705$id2,cl3705$id1,sep=";");
all1=all1[-c(37,39,28, 26,25,23,19,17,13,11,1,2)];
fullP7[match(idmatch,c(all,all1),nomatch=0)>0]=1;
#create graph
g7 = make_empty_graph(n, directed = FALSE) # empty graph
g7 = set_vertex_attr(g7, "id", value = 1:n)
g7 = add_edges(g7, as.vector(t(pairsf$pairs[fullP7==2,1:2])));
g7 = simplify(g7,remove.multiple = TRUE, remove.loops = TRUE)
clustg7 <- components(g7, "weak") # get clusters
blocksg7 <- data.frame(id=V(g7)$id, block=clustg7$membership) # block number
tbg7 = table(blocksg7$block);
big7bl = names(tbg7[tbg7>2]);
bjc = blocksg7[match(blocksg7$block,big7bl,nomatch=0)>0,];
crrf = c();
for (bl in biggbl){
ids = combn(bjc[bjc$block==bl,"id"],2);
crrf = rbind(crrf,t(ids));
crrf = rbind(crrf,t(ids[c(2,1),]));
}
fullP7t = fullP7
#make transitive closure
fullP7t[match(idmatch,paste(crrf[,1],crrf[,2],sep=";"))>0] = 2;
prsc = pairsf$pairs[fullP7t==2,1:2];
fullP7t[match(idmatch,paste(prsc[,2],prsc[,1],sep=";"))>0] = 2;
#write.table(fullP7t, file="/home/audris/fullP7t.csv",row.names=F,col.names=F);
#write.table(blocksg7[,2],file="/home/audris/fullP7tb.csv",row.names=F,col.names=F);
nn=list();
for (bl in names(tbg7[tbg7>9])){
sl = blocksg7[blocksg7$block==bl,"id"];
nn[[bl]]=cbind((1:n)[sl],pairsf$data1$a[sl]);
}
fullP7.c = fullP7;
#now use sadikas input to propagate back to fullP7
reclust = read.csv("RF_CorrectedLabels.csv")
toDisconect = c();
toConnect = c();
for (cl in names(table(reclust$cl))){
#first disconnect all
ids = reclust[reclust$cl==cl,"V1"];
ids = combn(ids,2);
ids = rbind(t(ids),t(ids[c(2,1),]));
toDisconect = rbind(toDisconect, ids);
#now reconnect subclusters
cl1s = names(table(reclust$l[reclust$cl==cl]));
for (cl1 in names(table(reclust$l[reclust$cl==cl]))){
ids = reclust[reclust$cl==cl&reclust$l==cl1,"V1"];
if (length(ids)>1){
ids = combn(ids,2);
ids = rbind(t(ids),t(ids[c(2,1),]));
toConnect = rbind(toConnect, ids);
}
}
}
fullP7.c[match(idmatch,paste(toDisconect[,1],toDisconect[,2],sep=";"),nomatch=0)>0]=1;
fullP7.c[match(idmatch,paste(toConnect[,1],toConnect[,2],sep=";"),nomatch=0)>0]=2;
g7c = make_empty_graph(n, directed = FALSE) # empty graph
g7c = set_vertex_attr(g7c, "id", value = 1:n)
g7c = add_edges(g7c, as.vector(t(pairsf$pairs[fullP7.c==2,1:2])));
g7c = simplify(g7c,remove.multiple = TRUE, remove.loops = TRUE)
clustg7c <- components(g7c, "weak") # get clusters
blocksg7c <- data.frame(id=V(g7c)$id, block=clustg7c$membership) # block number
tbg7c = table(blocksg7c$block);
big7cbl = names(tbg7c[tbg7c>2]);
bjc = blocksg7c[match(blocksg7c$block,big7cbl,nomatch=0)>0,];
crrf = c();
for (bl in big7cbl){
ids = combn(bjc[bjc$block==bl,"id"],2);
crrf = rbind(crrf,t(ids));
crrf = rbind(crrf,t(ids[c(2,1),]));
}
#make transitive closure
fullP7.c[match(idmatch,paste(crrf[,1],crrf[,2],sep=";"))>0] = 2;
save(fullP7.c, nn, tbg7c, blocksg7c, file = "/home/audris/rfmodelsFullP7.c.RData");
##################################
# compare to bitergia FINAL
##################################
library(readr)
library(igraph)
bitr <- read_delim("../BITERGIA/map2mysqlIds1", delim = ';', col_names = F)
names(bitr) = c("a","id");
bitr$a0=bitr$a;
tba = table(bitr$a)
morRec = data.frame(a="",a0="",id=0);
for (dev in names(tba[tba>1])){
ids = sort(bitr [ bitr$a == dev, "id"]$id);
for (i in 2:length(ids)){
morRec = rbind(morRec, data.frame(a=dev,a0=paste (dev,ids[i],sep=";"), id=ids[i]));
}
}
morRec = morRec[-1,]
bitr=rbind(bitr,morRec);
#as=read.table("/data/delta/openstack.a.gz",sep=";",quote="",colClasses="character", header=F, strip.white=F)
mb = match(bitr$a, pairsf$data1$a, nomatch=0);
bitr = bitr[mb>0,]
mb = match(bitr$a, pairsf$data1$a, nomatch=0);
#nowcreate graphs
tbi = table(bitr$id);
tba = table(bitr$a);
length(tbi)
8840
length(tba)
9028
nb=dim(bitr)[1];
bitr$off=1:nb;
gb <- make_empty_graph(nb, directed = FALSE); # empty graph
gb <- set_vertex_attr(gb, "a", value = bitr$a);
gb <- set_vertex_attr(gb, "id", value = bitr$id);
gb <- set_vertex_attr(gb, "i", value = 1:nb);
gn = gb;
for (dev in names(tbi[tbi>1])){
ids = bitr [ bitr$id == dev, "off"]$off;
for (i in 2:length(ids)){
gb <- add_edges(gb, c(ids[1],ids[i])) # candidate edges
}
}
for (dev in names(tba[tba>1])){
ids = combn(bitr [ bitr$a == dev, "off"]$off, 2);
ids = rbind(t(ids),t(ids[c(2,1),]));
gn <- add_edges(gn, t(ids));
}
prs=pairsf$pairs[fullP7.c==2,1:2];
prs1 = cbind(pairsf$data1[prs[,1],"a"],pairsf$data1[prs[,2],"a"]);
sel = match(as.vector(unclass(V(gn)$a)),prs1[,1],nomatch=0)>0 & match(as.vector(unclass(V(gn)$a)),prs1[,2],nomatch=0)>0
sel1 = match(prs1[,1],as.vector(unclass(V(gn)$a))[sel],nomatch=0)>0 & match(prs1[,2],as.vector(unclass(V(gn)$a))[sel],nomatch=0)>0
fr=(1:nb)[sel][match(prs1[sel1,1],as.vector(unclass(V(gn)$a))[sel],nomatch=0)];
to=(1:nb)[sel][match(prs1[sel1,2],as.vector(unclass(V(gn)$a))[sel],nomatch=0)];
gn <- add_edges(gn, as.vector(rbind(fr,to)))
gbf = as_long_data_frame(gb)[,c("from_a","to_a")];
gnf = as_long_data_frame(gn)[,c("from_a","to_a")];
sum(is.na(match(paste(gbf[,1],gbf[,2],sep=";"),paste(gnf[,1],gnf[,2],sep=";"))))
[1] 6
> sum(is.na(match(paste(gnf[,1],gnf[,2],sep=";"),paste(gbf[,1],gbf[,2],sep=";"))))
[1] 17587
dim(V(gn))
10344
dim(gnf)
[1] 22485 2
dim(gbf)
1504
measures = function (p, t){
tt=table(p,t);
rec = tt[2,2]/(tt[2,2]+tt[1,2]);
prec = tt[2,2]/(tt[2,2]+tt[2,1]);
spl = 1-rec;
lu = tt[2,1]/(tt[2,2]+tt[1,2]);
c(prec, rec, spl, lu);
}
clustb <- components(gb, "weak") # get clusters
blocksb <- data.frame(id = V(gb)$i, block=clustb$membership) # block number
gbc = contract(gb,blocksb[,"block"],vertex.attr.comb=list(i="min"));
clustn <- components(gn, "weak") # get clusters
blocksn <- data.frame(id = V(gn)$i, block=clustn$membership) # block number
gnc = contract(gn,blocksn[,"block"],vertex.attr.comb=list(i="min"));
gnc=simplify(gnc,remove.multiple = TRUE, remove.loops = TRUE)
gbc=simplify(gbc,remove.multiple = TRUE, remove.loops = TRUE)
tn = table(blocksn$block)
tb = table(blocksb$block)
fwrite(blocksn,"/home/audris/aaBOS.txt",sep=";",quote=F, col.names=F)
fwrite(blocksb,"/home/audris/bitBOS.txt",sep=";",quote=F, col.names=F)
#create complete graphs for each block and compare
#perl splCl.perl aaBOS.txt bitBOS.txt
read 10344
created la: 6271 clstrs with max 24
created lt: 8840 clstrs with max 7
tp=14828 fp=14
tp=14828;fp=14;fn=16802
c(tp/(tp + fp), tp/(tp+fn), fn/(tp + fn), fp/(tp + fn))
[1] 0.9990567309 0.4687954474 0.5312045526 0.0004426178
length(tba)
length(tn)
6271
badSplit = c();
for (id in names(tn[tn>1])){
ids = blocksb[match(blocksb[,1],blocksn[blocksn$block==id,1],nomatch=0)>0,2];
badSplit = c(badSplit,length(table(ids)));
}
badClump = c();
for (id in names(tn)){
cl = blocksb[blocksb$id==id,2];
ids = blocksb[blocksb$block==cl,1];
badClump = c(badClump,length(table(ids)));
}
table(badSplit>1)
FALSE TRUE
141 1641
1641/nb
[1] 0.1586427
(sum(badSplit)-length(badSplit))/nb
[1] 0.2488399
table(badClump>1)
FALSE TRUE
4937 1334
1334/nb
[1] 0.1289637
nb