erp5_wendelin_data_lake_ingestion: script to get size of all data lake and by dataset

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_checkIngestedData.py

+import json
+portal = context.getPortalObject()
+portal_catalog = portal.portal_catalog
+
+def getDatasetInfo(data_set):
+  size = 0
+  datastream_result_dict = json.loads(portal.ERP5Site_getDataStreamList(data_set.getReference()))
+  for stream_dict in datastream_result_dict['result']:
+    size += stream_dict['full-size']
+  return len(datastream_result_dict['result']), size
+
+def format_size(num, suffix='b'):
+  for unit in ['','K','M','G','T','P','E','Z']:
+    if abs(num) < 1024.0:
+      return "%3.1f %s%s" % (num, unit, suffix)
+    num /= 1024.0
+  return "%.1f %s%s" % (num, 'Yi', suffix)
+
+data_set_list = []
+
+if data_set_reference:
+  try:
+    data_set = portal.data_set_module.get(data_set_reference)
+    if data_set is None or portal.ERP5Site_checkReferenceInvalidated(data_set):
+      return "Not found: there is no valid dataset for that reference"
+    data_set_list.append(data_set)
+  except Exception as e: # fails because unauthorized access
+    return "ERROR: " + str(e)
+else:
+  data_set_list = portal_catalog(portal_type="Data Set", validation_state='validated OR published')
+
+total_size = 0
+for data_set in data_set_list:
+  print "Data set " + data_set.getReference()
+  nfiles, size = getDatasetInfo(data_set)
+  total_size += size
+  print "  #files: " + str(nfiles)
+  print "  Size: " + format_size(size)
+  print
+
+if len(data_set_list) > 1:
+  print
+  print "TOTAL SIZE: " + format_size(total_size)
+
+return printed

bt5/erp5_wendelin_data_lake_ingestion/SkinTemplateItem/portal_skins/erp5_wendelin_data_lake/ERP5Site_checkIngestedData.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>Script_magic</string> </key>
+            <value> <int>3</int> </value>
+        </item>
+        <item>
+            <key> <string>_bind_names</string> </key>
+            <value>
+              <object>
+                <klass>
+                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
+                </klass>
+                <tuple/>
+                <state>
+                  <dictionary>
+                    <item>
+                        <key> <string>_asgns</string> </key>
+                        <value>
+                          <dictionary>
+                            <item>
+                                <key> <string>name_container</string> </key>
+                                <value> <string>container</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_context</string> </key>
+                                <value> <string>context</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_m_self</string> </key>
+                                <value> <string>script</string> </value>
+                            </item>
+                            <item>
+                                <key> <string>name_subpath</string> </key>
+                                <value> <string>traverse_subpath</string> </value>
+                            </item>
+                          </dictionary>
+                        </value>
+                    </item>
+                  </dictionary>
+                </state>
+              </object>
+            </value>
+        </item>
+        <item>
+            <key> <string>_params</string> </key>
+            <value> <string>data_set_reference=None</string> </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>ERP5Site_checkIngestedData</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>