Update abstract.py

abstract.py

@@ -4,21 +4,41 @@ import numpy as np
 from rank_bm25 import BM25Okapi
 from nltk.tokenize import word_tokenize
 import nltk
+import sys
+
 nltk.download('punkt')
 
-# Load data from the PKL file
-with open('data_hydrogen.pkl', 'rb') as pkl_file:
-    data = pickle.load(pkl_file)
+def load_data(file_path):
+    with open(file_path, 'rb') as pkl_file:
+        data = pickle.load(pkl_file)
+    return data
+
+def save_texts(data, output_path='texts.npy'):
+    np.save(output_path, np.array(data['Abstract'].values))
+
+def load_texts(file_path='texts.npy'):
+    return np.load(file_path, allow_pickle=True)
 
-# save texts as .npy file
-np.save('texts.npy', np.array(data['Abstract'].values))
+def tokenize_texts(texts):
+    return [word_tokenize(doc.lower()) for doc in texts]
 
+def build_bm25_model(tokenized_texts):
+    return BM25Okapi(tokenized_texts)
 
-loaded_texts = np.load('texts.npy', allow_pickle=True)
+def save_bm25_model(bm25_model, output_path='bm25.pkl'):
+    with open(output_path, 'wb') as pkl_file:
+        pickle.dump(bm25_model, pkl_file)
 
-tokenized_texts = [word_tokenize(doc.lower()) for doc in loaded_texts]
-bm25 = BM25Okapi(tokenized_texts)
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python script.py <file_path>")
+        sys.exit(1)
 
-with open('bm25.pkl', 'wb') as pkl_file:
-    pickle.dump(bm25, pkl_file)
+    file_path = sys.argv[1]
 
+    data = load_data(file_path)
+    save_texts(data)
+    loaded_texts = load_texts()
+    tokenized_texts = tokenize_texts(loaded_texts)
+    bm25 = build_bm25_model(tokenized_texts)
+    save_bm25_model(bm25)