Upload dashboard.py

dashboard.py

@@ -35,7 +35,14 @@ hv.extension('bokeh')
 
 ## LOAD DATASETS
 
-dna_folder = './data'
+data_folder = './data'
+
+country_name_df = pd.read_csv(os.path.join(data_folder, 'country_name_map.tsv'),  header=0, sep='\t', lineterminator='\n', low_memory=False)
+country_name_map = dict(zip(country_name_df.Country_Code, country_name_df.Country_Name))
+
+total_publications_time_indexed = pd.read_csv(os.path.join(data_folder, 'total_publications_time_indexed.tsv'),  header=0, sep='\t', lineterminator='\n', low_memory=False)
+country_publications_time_indexed = pd.read_csv(os.path.join(data_folder, 'country_publications_time_indexed.tsv'),  header=0, sep='\t', lineterminator='\n', low_memory=False)
+
 
 ## AECO topic over time html file:
 AECO_topics_over_time_file_path = '/assets/optimized_merged_AECO_topics_over_time_2D.html'
@@ -43,18 +50,6 @@ AECO_topics_dendogram_file_path = '/assets/topic_hierarchy_optimal_params.htm'
 AECO_topic_map_path = '/assets/document_datamap_ver0.html'
 
 
-#### full data unfiltered:
-
-dna_articles_unfiltered_eu_time_indexed_resampled = pd.read_csv(os.path.join(dna_folder, 'dna_articles_unfiltered_eu_time_indexed_resampled.tsv'),sep='\t',header=0)
-dna_articles_unfiltered_us_time_indexed_resampled = pd.read_csv(os.path.join(dna_folder, 'dna_articles_unfiltered_us_time_indexed_resampled.tsv'),sep='\t',header=0)
-dna_articles_unfiltered_eu_us_time_indexed_resampled = pd.read_csv(os.path.join(dna_folder, 'dna_articles_unfiltered_eu_us_time_indexed_resampled.tsv'),sep='\t',header=0)
-
-#### classifier filtered articles:
-
-dh_ration_df_eu = pd.read_csv(os.path.join(dna_folder, 'dh_ration_df_eu.tsv'),sep='\t',header=0)
-dh_ration_df_us = pd.read_csv(os.path.join(dna_folder, 'dh_ration_df_us.tsv'),sep='\t',header=0)
-dh_ration_df_eu_us = pd.read_csv(os.path.join(dna_folder, 'dh_ration_df_eu_us.tsv'),sep='\t',header=0)
-
 regions = ['eu', 'us', 'eu_us']
 
 sorted_ent_type_freq_map_eu=dict()
@@ -62,19 +57,19 @@ sorted_ent_type_freq_map_us=dict()
 sorted_ent_type_freq_map_eu_us=dict()
 
 def read_top_ent_types():
-    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_type_freq_map_eu.tsv'), 'r'))
+    reader = csv.reader(open(os.path.join(data_folder, 'sorted_ent_type_freq_map_eu.tsv'), 'r'))
     for i,row in enumerate(reader):
         if i < 20:
             k, v = row
             sorted_ent_type_freq_map_eu[k] = int(v)
     del sorted_ent_type_freq_map_eu['Entity']
-    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_type_freq_map_us.tsv'), 'r'))
+    reader = csv.reader(open(os.path.join(data_folder, 'sorted_ent_type_freq_map_us.tsv'), 'r'))
     for i, row in enumerate(reader):
         if i < 20:
             k, v = row
             sorted_ent_type_freq_map_us[k] = int(v)
     del sorted_ent_type_freq_map_us['Entity']
-    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_type_freq_map_eu_us.tsv'), 'r'))
+    reader = csv.reader(open(os.path.join(data_folder, 'sorted_ent_type_freq_map_eu_us.tsv'), 'r'))
     for i, row in enumerate(reader):
         if i < 20:
             k, v = row
@@ -87,12 +82,9 @@ top_type_filtered_eu = ['DBpedia:Country', 'DBpedia:Organisation', 'DBpedia:Comp
 top_type_filtered_us = ['DBpedia:Organisation', 'DBpedia:Company', 'DBpedia:Disease', 'DBpedia:ChemicalSubstance', 'DBpedia:Person', 'DBpedia:Drug', 'DBpedia:Country', 'DBpedia:Region', 'DBpedia:MonoclonalAntibody', 'DBpedia:City', 'DBpedia:Biomolecule']
 top_type_filtered_eu_us =  ['DBpedia:Organisation', 'DBpedia:Company', 'DBpedia:ChemicalSubstance', 'DBpedia:Drug', 'DBpedia:Country', 'DBpedia:Person', 'DBpedia:Disease', 'DBpedia:MonoclonalAntibody', 'DBpedia:GovernmentAgency', 'DBpedia:Biomolecule', 'DBpedia:Gene']
 
-dna_healthtech_articles_eu_time_indexed_resampled=pd.read_csv(os.path.join(dna_folder, 'dna_healthtech_articles_eu_time_indexed_resampled.tsv'),sep='\t',header=0)
-dna_healthtech_articles_us_time_indexed_resampled=pd.read_csv(os.path.join(dna_folder, 'dna_healthtech_articles_us_time_indexed_resampled.tsv'),sep='\t',header=0)
-dna_healthtech_articles_eu_us_time_indexed_resampled=pd.read_csv(os.path.join(dna_folder, 'dna_healthtech_articles_eu_us_time_indexed_resampled.tsv'),sep='\t',header=0)
 
 def read_top_ent_maps():
-    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_freq_map_eu.tsv'), 'r'), delimiter='\t')
+    reader = csv.reader(open(os.path.join(data_folder, 'sorted_ent_freq_map_eu.tsv'), 'r'), delimiter='\t')
     for row in reader:
         k,v = row
         lista = ast.literal_eval(v)
@@ -102,7 +94,7 @@ def read_top_ent_maps():
         dizionario = sorted(dizionario.items(), key=lambda x: x[1], reverse=True)
         ent_freq_maps_eu[k]=dizionario
 
-    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_freq_map_us.tsv'), 'r'), delimiter='\t')
+    reader = csv.reader(open(os.path.join(data_folder, 'sorted_ent_freq_map_us.tsv'), 'r'), delimiter='\t')
     for row in reader:
         k, v = row
         lista = ast.literal_eval(v)
@@ -112,7 +104,7 @@ def read_top_ent_maps():
         dizionario = sorted(dizionario.items(), key=lambda x: x[1], reverse=True)
         ent_freq_maps_us[k] = dizionario
 
-    reader = csv.reader(open(os.path.join(dna_folder, 'sorted_ent_freq_map_eu_us.tsv'), 'r'), delimiter='\t')
+    reader = csv.reader(open(os.path.join(data_folder, 'sorted_ent_freq_map_eu_us.tsv'), 'r'), delimiter='\t')
     for row in reader:
         k, v = row
         lista = ast.literal_eval(v)
@@ -131,15 +123,15 @@ read_top_ent_maps()
 
 def read_type_filtered_triples():
     for t in top_type_filtered_eu:
-        df = pd.read_csv(dna_folder+'/filtered_rows/eu/'+t.replace(':','_')+'.tsv', sep="	", header=0)
+        df = pd.read_csv(data_folder+'/filtered_rows/eu/'+t.replace(':','_')+'.tsv', sep="	", header=0)
         df.drop(columns=['Unnamed: 0'], inplace=True)
         top_type_filtered_triples_eu[t]=df
     for t in top_type_filtered_us:
-        df = pd.read_csv(dna_folder+'/filtered_rows/us/'+t.replace(':','_')+'.tsv', sep="	")
+        df = pd.read_csv(data_folder+'/filtered_rows/us/'+t.replace(':','_')+'.tsv', sep="	")
         df.drop(columns=['Unnamed: 0'], inplace=True)
         top_type_filtered_triples_us[t]=df
     for t in top_type_filtered_eu_us:
-        df = pd.read_csv(dna_folder+'/filtered_rows/eu_us/'+t.replace(':','_')+'.tsv', sep="	")
+        df = pd.read_csv(data_folder+'/filtered_rows/eu_us/'+t.replace(':','_')+'.tsv', sep="	")
         df.drop(columns=['Unnamed: 0'], inplace=True)
         top_type_filtered_triples_eu_us[t]=df
 
@@ -151,18 +143,25 @@ top_type_filtered_triples_eu_us = dict()
 
 read_type_filtered_triples()
 
-grouping_filtered = pd.read_csv(os.path.join(dna_folder, 'dna_relations.tsv'), sep="	")
+grouping_filtered = pd.read_csv(os.path.join(data_folder, 'dna_relations.tsv'), sep="	")
+
+
+
 ################################# CREATE CHARTS ############################
-def create_curve_chart():
-    # Create the 3 line plots
-    curve_eu = hv.Curve((dh_ration_df_eu.index, dh_ration_df_eu.ids/dna_articles_unfiltered_eu_time_indexed_resampled.ids), 'Time', 'Digital Health News Ratio',label='EU')
-    curve_us = hv.Curve((dh_ration_df_us.index, dh_ration_df_us.ids/dna_articles_unfiltered_us_time_indexed_resampled.ids),'Time', 'Digital Health News Ratio', label='US')
-    curve_eu_us = hv.Curve((dh_ration_df_eu_us.index, dh_ration_df_eu_us.ids/dna_articles_unfiltered_eu_us_time_indexed_resampled.ids),'Time', 'Digital Health News Ratio', label='EU-US')
+def create_publication_curve_chart():
+    total_publications_time_indexed.id = np.log1p(total_publications_time_indexed.id)
+    country_publications_time_indexed = country_publications_time_indexed.applymap(lambda x: np.log1p(x) if np.issubdtype(type(x), np.number) else x)
+    curve_total = hv.Curve((total_publications_time_indexed.index, total_publications_time_indexed.id), 'Time', 'Publication Counts (log)',label='Total')
     #Overlay the line plots
-    overlay = curve_eu * curve_us * curve_eu_us
-    overlay.opts(show_legend = True, legend_position='top_left', width=1200, height=600)
+    overlay = curve_total
+    curve_countries = []
+    for country in country_name_map.keys():
+       overlay = overlay * hv.Curve((country_publications_time_indexed.index, country_publications_time_indexed[country]), label=country_name_map[country])
+    overlay.opts(show_legend=True,legend_position='right', width=800, height=400 )
     return overlay
 
+
+
 macro_topics = ["Energy-Efficient Building Design for Thermal Comfort and Sustainability","Indoor Air Quality and Energy Efficiency in Low-Energy Houses","Urban Planning and Development in China\'s Cities", "Design Thinking and Sustainable Product Development", "Smart Cities and Urban Computing", "Urban Resilience and Water Management","Renewable Energy Systems: Solar PV & Building Applications","Exploring the Intersection of Traditional Heritage and Modern Steel Architecture in Historical Buildings","Green Building Assessment and Design","Landscape Design, Planning, and Research: Integrating Cultural, Ecological, and Rural Perspectives", "Noise and Acoustic Design in Urban Development","Sustainable Building Materials: Wood & 3D Printing Innovations","BIM in AEC: Trends, Challenges, and Opportunities","Urban Food Systems: Community Development and Social Sustainability in Cities","Innovative Bridge Design and Construction: Trends and Case Studies", "Cavity Flow and Heat Transfer"]
 macro_topics_active_subset = ["Energy-Efficient Building Design for Thermal Comfort and Sustainability","Indoor Air Quality and Energy Efficiency in Low-Energy Houses","Smart Cities and Urban Computing", "Renewable Energy Systems: Solar PV & Building Applications","BIM in AEC: Trends, Challenges, and Opportunities"]
 
@@ -211,56 +210,6 @@ def load_country_network(topic, **kwargs):
         return html_pane
 
         
-# Define a function to generate Curve based on selected values
-def generate_entity_curves(region_value, type_value, **kwargs):
-  if region_value=='eu':
-    top20Ents = ent_freq_maps_eu[type_value]
-    curveList = []
-    for ent in top20Ents:
-      entityTriples =  top_type_filtered_triples_eu[type_value][(top_type_filtered_triples_eu[type_value]['subjEntityLinks']==ent[0]) | (top_type_filtered_triples_eu[type_value]['objEntityLinks']==ent[0])]
-      entityTriples_time_indexed = entityTriples.set_index(pd.DatetimeIndex(entityTriples['timestamp']), inplace=False)
-      del entityTriples_time_indexed['timestamp']
-      entityTriples_time_indexed_resampled = entityTriples_time_indexed.resample("Y").count()
-      #print(entityTriples_time_indexed_resampled)
-      entityTriples_time_indexed_resampled = entityTriples_time_indexed_resampled.reindex(dna_healthtech_articles_eu_time_indexed_resampled.index, fill_value=0)
-      curve = hv.Curve((entityTriples_time_indexed_resampled.index, (entityTriples_time_indexed_resampled['doc_id']/dna_healthtech_articles_eu_time_indexed_resampled['ids'])), 'Time', 'Key Entity Occurrence', label=ent[0])
-      curve.opts(autorange='y')
-      #curve.opts(logy=True)
-      curveList.append(curve)
-    overlay = hv.Overlay(curveList)
-    overlay.opts(legend_muted=False, legend_cols=4, show_legend = True, legend_position='top_left', fontsize={'legend':13},width=1200, height=800)
-    return overlay
-
-  elif region_value=='us':
-    top20Ents = ent_freq_maps_us[type_value]
-    curveList = []
-    for ent in top20Ents:
-      entityTriples =  top_type_filtered_triples_us[type_value][(top_type_filtered_triples_us[type_value]['subjEntityLinks']==ent[0]) | (top_type_filtered_triples_us[type_value]['objEntityLinks']==ent[0])]
-      entityTriples_time_indexed = entityTriples.set_index(pd.DatetimeIndex(entityTriples['timestamp']), inplace=False)
-      del entityTriples_time_indexed['timestamp']
-      entityTriples_time_indexed_resampled = entityTriples_time_indexed_resampled.reindex(dna_healthtech_articles_us_time_indexed_resampled.index, fill_value=0)
-      curve = hv.Curve((entityTriples_time_indexed_resampled.index, (entityTriples_time_indexed_resampled['doc_id']/dna_healthtech_articles_us_time_indexed_resampled['ids'])), 'Time', 'Key Entity Occurrence', label=ent[0])
-      curve.opts(autorange='y')
-      curveList.append(curve)
-    overlay = hv.Overlay(curveList)
-    overlay.opts(legend_muted=False, legend_cols=4, show_legend = True, legend_position='top_left', fontsize={'legend':13},width=1200, height=800)
-    return overlay
-
-  elif region_value=='eu_us':
-    top20Ents = ent_freq_maps_eu_us[type_value]
-    curveList = []
-    for ent in top20Ents:
-      entityTriples =  top_type_filtered_triples_eu_us[type_value][(top_type_filtered_triples_eu_us[type_value]['subjEntityLinks']==ent[0]) | (top_type_filtered_triples_eu_us[type_value]['objEntityLinks']==ent[0])]
-      entityTriples_time_indexed = entityTriples.set_index(pd.DatetimeIndex(entityTriples['timestamp']), inplace=False)
-      del entityTriples_time_indexed['timestamp']
-      entityTriples_time_indexed_resampled = entityTriples_time_indexed_resampled.reindex(dna_healthtech_articles_eu_us_time_indexed_resampled.index, fill_value=0)
-      curve = hv.Curve((entityTriples_time_indexed_resampled.index, (entityTriples_time_indexed_resampled['doc_id']/dna_healthtech_articles_eu_us_time_indexed_resampled['ids'])), 'Time', 'Key Entity Occurrence', label=ent[0])
-      curve.opts(autorange='y')
-      curveList.append(curve)
-    overlay = hv.Overlay(curveList)
-    overlay.opts(legend_muted=False, legend_cols=4, show_legend = True, legend_position='top_left', fontsize={'legend':13},width=1200, height=800)
-    return overlay
-
 
 ############################# WIDGETS & CALLBACK ###########################################
 
@@ -321,11 +270,13 @@ def generate_radio_buttons(value):
 
 # https://tabler-icons.io/
 button0 = pn.widgets.Button(name="Introduction", button_type="warning", icon="file-info", styles={"width": "100%"})
-button1 = pn.widgets.Button(name="Topic Map", button_type="warning", icon="chart-dots-3", styles={"width": "100%"})
-button2 = pn.widgets.Button(name="AECO Macro Topics Hierarchy", button_type="warning", icon="file-info", styles={"width": "100%"})
-button3 = pn.widgets.Button(name="AECO Macro Topics Trends", button_type="warning",  icon="chart-histogram", styles={"width": "100%"})
-button4 = pn.widgets.Button(name="Research Collaboration Networks: Institutes", button_type="warning", icon="chart-dots-3", styles={"width": "100%"})
-button5 = pn.widgets.Button(name="Research Collaboration Networks: Countries", button_type="warning", icon="chart-dots-3", styles={"width": "100%"})
+button1 = pn.widgets.Button(name="Publication Trends", button_type="warning", icon="chart-histogram", styles={"width": "100%"})
+button2 = pn.widgets.Button(name="Topic Map", button_type="warning", icon="chart-dots-3", styles={"width": "100%"})
+button3 = pn.widgets.Button(name="AECO Macro Topics Hierarchy", button_type="warning",  icon="chart-dots-3", styles={"width": "100%"})
+button4 = pn.widgets.Button(name="AECO Macro Topics Trends", button_type="warning", icon="chart-histogram", styles={"width": "100%"})
+button5 = pn.widgets.Button(name="Research Collaboration Networks: Institutes", button_type="warning", icon="chart-dots-3", styles={"width": "100%"})
+button6 = pn.widgets.Button(name="Research Collaboration Networks: Countries", button_type="warning", icon="chart-dots-3", styles={"width": "100%"})
+
 
 region1 = pn.widgets.RadioButtonGroup(name='### Select News Region', options=regions)
 
@@ -347,7 +298,6 @@ def update_radio_group(event):
 
 
 # bind the function to the widget(s)
-dmap2 = hv.DynamicMap(pn.bind(generate_entity_curves, radio_buttons_regions,radio_buttons_types))
 # Bind the selected value of the first RadioButtonGroup to update the second RadioButtonGroup
 radio_buttons_regions.param.watch(update_radio_group, 'value')
 
@@ -375,7 +325,7 @@ button2.on_click(lambda event: show_page("Page2"))
 button3.on_click(lambda event: show_page("Page3"))
 button4.on_click(lambda event: show_page("Page4"))
 button5.on_click(lambda event: show_page("Page5"))
-#button5.on_click(lambda event: show_page("Page5"))
+button6.on_click(lambda event: show_page("Page6"))
 #button6.on_click(lambda event: show_page("Page6"))
 
 
@@ -402,9 +352,16 @@ for the 16 macro-topics automatically detected by an optimized BerTopic model an
 ### Research Collaboration Networks: Authors
 """, width=800), align="center")
 
+def CreatePage1():
+    return pn.Column(
+        pn.pane.Markdown("## Publication Trends "),
+        create_publication_curve_chart(),
+        align="center",
+    )
 
 
-def CreatePage1():
+
+def CreatePage2():
     # Load the HTML content from the local file
     #with open(AECO_topics_over_time_file_path, 'r', encoding='utf-8') as file:
     #    html_content = file.read()
@@ -415,7 +372,7 @@ def CreatePage1():
     return pn.Column(pn.pane.Markdown(" ## AECO Topic Map "), html_pane, align="center")
 
     
-def CreatePage2():
+def CreatePage3():
     # Load the HTML content from the local file
     #with open(AECO_topics_over_time_file_path, 'r', encoding='utf-8') as file:
     #    html_content = file.read()
@@ -426,7 +383,7 @@ def CreatePage2():
     return pn.Column(pn.pane.Markdown(" ## AECO Macro Topics Dendogram "), html_pane, align="center")
 
 
-def CreatePage3():
+def CreatePage4():
     # Load the HTML content from the local file
     #with open(AECO_topics_over_time_file_path, 'r', encoding='utf-8') as file:
     #    html_content = file.read()
@@ -436,7 +393,7 @@ def CreatePage3():
     html_pane = pn.pane.HTML(iframe_html , sizing_mode='stretch_both')
     return pn.Column(pn.pane.Markdown(" ## AECO Macro Topics "), html_pane, align="center")
 
-def CreatePage4():
+def CreatePage5():
     return pn.Column(
     macro_topics_button,
        pn.bind(load_institute_network, macro_topics_button),
@@ -444,7 +401,7 @@ def CreatePage4():
     )
 
 
-def CreatePage5():
+def CreatePage6():
     return pn.Column(
     macro_topics_button,
        pn.bind(load_country_network, macro_topics_button),
@@ -452,13 +409,6 @@ def CreatePage5():
     )
 
 
-def CreatePage6():
-    return pn.Column(
-        pn.pane.Markdown("## Entity Chord Diagrams "),
-        pn.Row(region_radio_button, pn.bind(filter_region, region_radio_button)),
-        align="center", )
-
-
 def CreatePage6():
     html = """<iframe src="https://app.vosviewer.com/?json=https%3A%2F%2Fdrive.google.com%2Fuc%3Fid%3D16q1oLQyEeMosAgeD9UkC9hSrpzAYX_-n" width="800" height="800"></iframe>"""
 
@@ -478,12 +428,12 @@ mapping = {
     "Page2": CreatePage2(),
     "Page3": CreatePage3(),
     "Page4": CreatePage4(),
-    "Page5": CreatePage5()
-    #"Page6": CreatePage6()
+    "Page5": CreatePage5(),
+    "Page6": CreatePage6()
 }
 
 #################### SIDEBAR LAYOUT ##########################
-sidebar = pn.Column(pn.pane.Markdown("## Pages"),button0,button1,button2,button3,button4,button5,
+sidebar = pn.Column(pn.pane.Markdown("## Pages"),button0,button1,button2,button3,button4,button5,button6,
                     #button5,
                     #button6,
                     styles={"width": "100%", "padding": "15px"})