Coverage for SynRBL/SynUtils/data_utils.py: 16%
97 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-04-11 11:17 +0200
« prev ^ index » next coverage.py v7.4.3, created at 2024-04-11 11:17 +0200
1import json
2import random
4from typing import List, Dict, Set, Any
5from typing import Optional, Union, Callable, Tuple
6from collections import defaultdict
9def save_database(database: list[dict], pathname: str = "./Data/database.json") -> None:
10 """
11 Save a database (a list of dictionaries) to a JSON file.
13 Args:
14 database: The database to be saved.
15 pathname: The path where the database will be saved. Defaults to './Data/database.json'.
17 Raises:
18 TypeError: If the database is not a list of dictionaries.
19 ValueError: If there is an error writing the file.
20 """
21 if not all(isinstance(item, dict) for item in database):
22 raise TypeError("Database should be a list of dictionaries.")
24 try:
25 with open(pathname, "w") as f:
26 json.dump(database, f)
27 except IOError as e:
28 raise ValueError(f"Error writing to file {pathname}: {e}")
31def load_database(pathname: str = "./Data/database.json") -> List[Dict]:
32 """
33 Load a database (a list of dictionaries) from a JSON file.
35 Args:
36 pathname: The path from where the database will be loaded. Defaults to './Data/database.json'.
38 Returns:
39 The loaded database.
41 Raises:
42 ValueError: If there is an error reading the file.
43 """
44 try:
45 with open(pathname, "r") as f:
46 database = json.load(f) # Load the JSON data from the file
47 return database
48 except IOError as e:
49 raise ValueError(f"Error reading to file {pathname}: {e}")
52def extract_atomic_elements(rules: List[Dict[str, Dict[str, int]]]) -> Set[str]:
53 """
54 Extracts the set of all atomic elements from a list of rules.
56 Args:
57 rules: A list of rules, where each rule is a dictionary representing a
58 composition of atomic elements.
60 Returns:
61 A set of all atomic elements found in the rules.
63 Example:
64 ```python
65 rules = [{"Composition": {"A": 1, "B": 1}}, {"Composition": {"C": 1, "D": 1, "E": 1}}]
66 atomic_elements = extract_atomic_elements(rules)
67 print(atomic_elements) # Output: {'A', 'B', 'C', 'D', 'E'}
68 ```
69 """
71 atomic_elements = set()
73 for rule in rules:
74 atomic_elements.update(rule["Composition"].keys())
76 return atomic_elements
79def _get_max_comp_len(database: List[Dict]) -> int:
80 """
81 Determines the maximum length of a composition in the database.
83 Args:
84 database (List[Dict]): A list of dictionaries representing the database.
86 Returns:
87 int: The maximum length of a composition in the database.
89 Example:
90 ```python
91 database = [
92 {'Composition': ['H2O', 'CO2']},
93 {'Composition': ['NaCl', 'H2O']},
94 {'Composition': ['C6H12O6', 'H2O']},
95 ]
97 max_comp_len = _get_max_comp_len(database)
98 print(max_comp_len) # Output: 3
99 ```
100 """
102 max_comp_len = 0 # Initialize the maximum composition length
104 # Iterate through each entry in the database
105 for entry in database:
106 # Check if the current entry's composition length exceeds the maximum
107 if max_comp_len < len(entry["Composition"]):
108 # Update the maximum composition length if necessary
109 max_comp_len = len(entry["Composition"])
111 # Return the maximum composition length
112 return max_comp_len
115def find_shortest_sublists(solution: List[List[Dict]]) -> List[List[Dict]]:
116 """
117 Find all sublists of dictionaries that have the shortest length.
119 Args:
120 solution: A list containing lists of dictionaries.
122 Returns:
123 A list of all sublists with the shortest length.
124 """
125 if not solution:
126 return []
128 min_length = min(len(sublist) for sublist in solution)
129 shortest_sublists = [sublist for sublist in solution if len(sublist) == min_length]
131 return shortest_sublists
134def filter_data(
135 data: List[Dict[str, Any]],
136 unbalance_values: Optional[List[str]] = None,
137 formula_key: str = "Diff_formula",
138 element_key: Optional[str] = None,
139 min_count: int = 0,
140 max_count: int = float("inf"),
141) -> List[Dict[str, Any]]:
142 """
143 Filter dictionaries based on a list of unbalance values and element count in a specified formula key.
145 This function filters the input list of dictionaries based on the specified list of unbalance values and
146 the count of a specific element within a given formula key. It returns dictionaries that match any of the
147 unbalance criteria and where the element count falls within the specified range.
149 Args:
150 data: A list of dictionaries to be filtered.
151 unbalance_values: The values to filter by in the 'Unbalance' key. If None, this criterion is ignored.
152 formula_key: The key in the dictionaries that contains the element counts. Defaults to 'Diff_formula'.
153 element_key: The element to filter by in the formula key. If None, this criterion is ignored.
154 min_count: The minimum allowed count of the element. Defaults to 0.
155 max_count: The maximum allowed count of the element. Defaults to infinity.
157 Returns:
158 A list of dictionaries filtered based on the criteria.
159 """
160 filtered_data = []
162 for item in data:
163 # Check for unbalance condition
164 unbalance_matches = (
165 unbalance_values is None or item.get("Unbalance") in unbalance_values
166 )
168 # Check for element count condition
169 if element_key is None:
170 element_matches = True
171 else:
172 element_count = item.get(formula_key, {}).get(element_key, 0)
173 element_matches = min_count <= element_count <= max_count
175 if unbalance_matches and element_matches:
176 filtered_data.append(item)
178 return filtered_data
181def remove_duplicates_by_key(
182 data: List[dict], key_function: Callable[..., Any]
183) -> List[dict]:
184 """
185 Remove duplicate entries from a list based on a unique key for each entry.
187 Parameters:
188 - `data` (List[dict]): A list of data entries (dictionaries, objects, etc.).
189 - `key_function` (Callable[..., Any]): A function that takes an entry from `data` and returns a key for duplicate check.
191 Returns:
192 - `List[dict]`: A list of unique entries, based on the unique keys generated.
194 Example:
195 >>> data = [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}, {'name': 'Alice', 'age': 30}]
196 >>> remove_duplicates_by_key(data, lambda x: (x['name'], x['age']))
197 [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}]
198 """
200 seen_keys = set()
201 unique_data = []
203 for entry in data:
204 key = frozenset(key_function(entry))
205 if key not in seen_keys:
206 seen_keys.add(key)
207 unique_data.append(entry)
209 return unique_data
212def sort_by_key_length(
213 data: List[Any], key_function: Callable[[Any], Any]
214) -> List[Any]:
215 """
216 Sort a list of entries based on the length of a specific key.
218 Args:
219 - data (List[Any]): A list of data entries.
220 - key_function (Callable[[Any], Any]): A function that takes an entry from `data` and returns a key
221 whose length is to be used for sorting.
223 Returns:
224 - List[Any]: A list of entries sorted by the length of the specified key.
225 """
227 return sorted(data, key=lambda x: len(key_function(x)))
230def add_missing_key_to_dicts(
231 data: List[Dict[str, Dict[str, Any]]],
232 dict_key: str,
233 missing_key: str,
234 default_value: Any,
235) -> List[Dict[str, Dict[str, Any]]]:
236 """
237 Iterates through a list of dictionaries and adds a specified key with a default value to a specified
238 dictionary within each main dictionary, if the key is not already present. Returns a new list with the updates.
240 Args:
241 data: A list of dictionaries.
242 dict_key: The key in the main dictionaries that points to another dictionary where the check should be done.
243 missing_key: The key to add if it's not present in the nested dictionary.
244 default_value: The default value to assign to the missing key.
246 Returns:
247 A new list of dictionaries with the missing key added where necessary.
249 Example:
250 >>> data = [{'Composition': {'A': 1, 'B': 2}}, {'Composition': {'B': 3}}]
251 >>> updated_data = add_missing_key_to_dicts(data, 'Composition', 'Q', 0)
252 >>> updated_data
253 [{'Composition': {'A': 1, 'B': 2, 'Q': 0}}, {'Composition': {'B': 3, 'Q': 0}}]
254 """
256 updated_data = []
258 for entry in data:
259 # Create a copy of the entry to avoid modifying the original data
260 updated_entry = entry.copy()
262 # Check if the missing key is not in the specified dictionary
263 if missing_key not in updated_entry.get(dict_key, {}):
264 # Ensure the dict_key points to a dictionary
265 if dict_key not in updated_entry:
266 updated_entry[dict_key] = {}
268 # Add the missing key with the default value
269 updated_entry[dict_key][missing_key] = default_value
271 updated_data.append(updated_entry)
273 return updated_data
276def extract_results_by_key(
277 data: List[Dict[str, any]], key: str = "new_reaction"
278) -> Tuple[List[Dict[str, any]], List[Dict[str, any]]]:
279 """
280 Separate dictionaries from a list into two lists based on the presence of a specific key.
282 Args:
283 data: A list of dictionaries to be separated.
284 key: The key to check for in each dictionary. Defaults to 'new_reaction'.
286 Returns:
287 A tuple of two lists:
288 - The first list contains dictionaries that have the specified key.
289 - The second list contains dictionaries that do not have the specified key.
290 """
292 with_key: List[Dict[str, any]] = []
293 without_key: List[Dict[str, any]] = []
295 # Separate dictionaries based on the presence of the key
296 for item in data:
297 if key in item:
298 with_key.append(item)
299 else:
300 without_key.append(item)
302 return with_key, without_key
305def get_random_samples_by_key(
306 data: List[Dict[str, Any]],
307 stratify_key: str,
308 num_samples_per_group: int = 1,
309 random_seed: int = None,
310) -> List[Dict[str, Any]]:
311 """
312 Get random samples from data, grouped by a specified key.
314 Parameters:
315 - data: List of dictionaries containing various keys.
316 - stratify_key: The key used for stratifying the data.
317 - num_samples_per_group: Number of random samples to draw from each unique group.
318 - random_seed: Seed for the random number generator for reproducibility.
320 Returns:
321 - A list of randomly selected samples from each unique group based on the stratify_key.
322 """
323 if random_seed is not None:
324 random.seed(random_seed)
326 # Function to create a sortable key from the item
327 def sortable_key(item: Dict[str, Any]) -> Union[str, tuple]:
328 key_value = item.get(stratify_key, None)
329 if isinstance(key_value, dict):
330 return tuple(sorted(key_value.items()))
331 elif isinstance(key_value, list):
332 return tuple(sorted(key_value))
333 else:
334 return key_value
336 # Group data by the specified stratify_key
337 grouped_data = defaultdict(list)
338 for item in data:
339 grouped_key = sortable_key(item)
340 grouped_data[grouped_key].append(item)
342 # Select random samples from each group
343 random_samples = []
344 for _, items in grouped_data.items():
345 if len(items) >= num_samples_per_group:
346 random_samples.extend(random.sample(items, num_samples_per_group))
347 else:
348 # If there aren't enough items, take all available
349 random_samples.extend(items)
351 return random_samples