אצווה 32 קבועה

machine-learn/compar_f1score.py

@@ -1,6 +1,7 @@
 import spacy
 from spacy.training.example import Example
 from tabulate import tabulate
+import os
 
 # Function to evaluate model and return evaluation metrics
 def evaluate_model(model_name, data):
@@ -192,8 +193,13 @@ def evaluate_model(model_name, data):
 models_metrics = []
 
 # Iterate over model versions
-for i in range(1, 10):
-    model_name = f"custom_ner_model23-{i}git"
+model_names = [i for i in os.listdir() if "custom_ner_model" in i]
+
+
+
+
+
+for model_name in model_names:
     print(f"Evaluating {model_name}...")
     f1_score, precision, recall = evaluate_model(model_name, data)
     models_metrics.append({

machine-learn/creating_model_git.py

@@ -63,7 +63,6 @@ def custom_tokenizer(nlp):
             example = Example.from_dict(nlp.make_doc(example_text), {'entities': entities})
             training_data.append(example)
 
-random.shuffle(training_data)
 
 nlp.begin_training()
 
@@ -75,18 +74,20 @@ def custom_tokenizer(nlp):
 
 n_iter = 100
 batch_sizes = compounding(16.0, 64.0, 1.001)
+batch_size = 32
+drop_size = 0.4
 iteration_data = {}
 #initial_lr = 0.001  # שיעור למידה התחלתי
 #lr_decay = 0.95  # קצב דעיכת שיעור הלמידה
 # optimizer.learn_rate = initial_lr
 
 for itn in range(n_iter):
     random.shuffle(training_data)
-    batches = minibatch(training_data, size=batch_sizes)
     losses = {}
-    for batch in batches:
-        nlp.update(batch, drop=0.4, losses=losses)
-    print(f"Iteration {itn}, Losses: {losses}")
+    for i in range(0, len(training_data), batch_size):
+        batch = training_data[i:i + batch_size]
+        nlp.update(batch, drop=drop_size, losses=losses)
+    print(f"Iteration {itn}: {losses}")
     iteration_data[itn] = losses.copy()
 
     current_loss = losses.get('ner', float('inf'))