From 5d81f7ac16c9110a13c0fed8c9afe2ec07170eb9 Mon Sep 17 00:00:00 2001
From: Jindra Helcl <helcl@ufal.mff.cuni.cz>
Date: Sun, 26 Nov 2023 11:39:34 +0100
Subject: [PATCH] using python for removal of empty lines because it did not
 work

---
 opuscleaner/filters/remove_empty_lines.json |  4 ++--
 opuscleaner/filters/remove_empty_lines.py   | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100755 opuscleaner/filters/remove_empty_lines.py

diff --git a/opuscleaner/filters/remove_empty_lines.json b/opuscleaner/filters/remove_empty_lines.json
index 49b7029..3a4e491 100644
--- a/opuscleaner/filters/remove_empty_lines.json
+++ b/opuscleaner/filters/remove_empty_lines.json
@@ -1,6 +1,6 @@
 {
     "description": "Remove lines that are empty or just whitespace on either side",
     "type": "bilingual",
-    "command": "grep -avE $'^\\\\s*\\t|\\t\\\\s*$|^[^\\t]*$'",
+    "command": "./remove_empty_lines.py",
     "parameters": {}
-}
\ No newline at end of file
+}
diff --git a/opuscleaner/filters/remove_empty_lines.py b/opuscleaner/filters/remove_empty_lines.py
new file mode 100755
index 0000000..1c04240
--- /dev/null
+++ b/opuscleaner/filters/remove_empty_lines.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+import sys
+
+def main():
+    for line in sys.stdin:
+        fields = line.strip("\r\n").split("\t")
+        ok = True
+        for field in fields:
+            if not field.strip():
+                ok = False
+                break
+        if ok:
+            sys.stdout.write(line)
+
+if __name__ == '__main__':
+    main()