Coverage for SynRBL/SynUtils/data_utils.py: 16%

97 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-04-11 11:17 +0200

1import json 

2import random 

3 

4from typing import List, Dict, Set, Any 

5from typing import Optional, Union, Callable, Tuple 

6from collections import defaultdict 

7 

8 

9def save_database(database: list[dict], pathname: str = "./Data/database.json") -> None: 

10 """ 

11 Save a database (a list of dictionaries) to a JSON file. 

12 

13 Args: 

14 database: The database to be saved. 

15 pathname: The path where the database will be saved. Defaults to './Data/database.json'. 

16 

17 Raises: 

18 TypeError: If the database is not a list of dictionaries. 

19 ValueError: If there is an error writing the file. 

20 """ 

21 if not all(isinstance(item, dict) for item in database): 

22 raise TypeError("Database should be a list of dictionaries.") 

23 

24 try: 

25 with open(pathname, "w") as f: 

26 json.dump(database, f) 

27 except IOError as e: 

28 raise ValueError(f"Error writing to file {pathname}: {e}") 

29 

30 

31def load_database(pathname: str = "./Data/database.json") -> List[Dict]: 

32 """ 

33 Load a database (a list of dictionaries) from a JSON file. 

34 

35 Args: 

36 pathname: The path from where the database will be loaded. Defaults to './Data/database.json'. 

37 

38 Returns: 

39 The loaded database. 

40 

41 Raises: 

42 ValueError: If there is an error reading the file. 

43 """ 

44 try: 

45 with open(pathname, "r") as f: 

46 database = json.load(f) # Load the JSON data from the file 

47 return database 

48 except IOError as e: 

49 raise ValueError(f"Error reading to file {pathname}: {e}") 

50 

51 

52def extract_atomic_elements(rules: List[Dict[str, Dict[str, int]]]) -> Set[str]: 

53 """ 

54 Extracts the set of all atomic elements from a list of rules. 

55 

56 Args: 

57 rules: A list of rules, where each rule is a dictionary representing a 

58 composition of atomic elements. 

59 

60 Returns: 

61 A set of all atomic elements found in the rules. 

62 

63 Example: 

64 ```python 

65 rules = [{"Composition": {"A": 1, "B": 1}}, {"Composition": {"C": 1, "D": 1, "E": 1}}] 

66 atomic_elements = extract_atomic_elements(rules) 

67 print(atomic_elements) # Output: {'A', 'B', 'C', 'D', 'E'} 

68 ``` 

69 """ 

70 

71 atomic_elements = set() 

72 

73 for rule in rules: 

74 atomic_elements.update(rule["Composition"].keys()) 

75 

76 return atomic_elements 

77 

78 

79def _get_max_comp_len(database: List[Dict]) -> int: 

80 """ 

81 Determines the maximum length of a composition in the database. 

82 

83 Args: 

84 database (List[Dict]): A list of dictionaries representing the database. 

85 

86 Returns: 

87 int: The maximum length of a composition in the database. 

88 

89 Example: 

90 ```python 

91 database = [ 

92 {'Composition': ['H2O', 'CO2']}, 

93 {'Composition': ['NaCl', 'H2O']}, 

94 {'Composition': ['C6H12O6', 'H2O']}, 

95 ] 

96 

97 max_comp_len = _get_max_comp_len(database) 

98 print(max_comp_len) # Output: 3 

99 ``` 

100 """ 

101 

102 max_comp_len = 0 # Initialize the maximum composition length 

103 

104 # Iterate through each entry in the database 

105 for entry in database: 

106 # Check if the current entry's composition length exceeds the maximum 

107 if max_comp_len < len(entry["Composition"]): 

108 # Update the maximum composition length if necessary 

109 max_comp_len = len(entry["Composition"]) 

110 

111 # Return the maximum composition length 

112 return max_comp_len 

113 

114 

115def find_shortest_sublists(solution: List[List[Dict]]) -> List[List[Dict]]: 

116 """ 

117 Find all sublists of dictionaries that have the shortest length. 

118 

119 Args: 

120 solution: A list containing lists of dictionaries. 

121 

122 Returns: 

123 A list of all sublists with the shortest length. 

124 """ 

125 if not solution: 

126 return [] 

127 

128 min_length = min(len(sublist) for sublist in solution) 

129 shortest_sublists = [sublist for sublist in solution if len(sublist) == min_length] 

130 

131 return shortest_sublists 

132 

133 

134def filter_data( 

135 data: List[Dict[str, Any]], 

136 unbalance_values: Optional[List[str]] = None, 

137 formula_key: str = "Diff_formula", 

138 element_key: Optional[str] = None, 

139 min_count: int = 0, 

140 max_count: int = float("inf"), 

141) -> List[Dict[str, Any]]: 

142 """ 

143 Filter dictionaries based on a list of unbalance values and element count in a specified formula key. 

144 

145 This function filters the input list of dictionaries based on the specified list of unbalance values and 

146 the count of a specific element within a given formula key. It returns dictionaries that match any of the 

147 unbalance criteria and where the element count falls within the specified range. 

148 

149 Args: 

150 data: A list of dictionaries to be filtered. 

151 unbalance_values: The values to filter by in the 'Unbalance' key. If None, this criterion is ignored. 

152 formula_key: The key in the dictionaries that contains the element counts. Defaults to 'Diff_formula'. 

153 element_key: The element to filter by in the formula key. If None, this criterion is ignored. 

154 min_count: The minimum allowed count of the element. Defaults to 0. 

155 max_count: The maximum allowed count of the element. Defaults to infinity. 

156 

157 Returns: 

158 A list of dictionaries filtered based on the criteria. 

159 """ 

160 filtered_data = [] 

161 

162 for item in data: 

163 # Check for unbalance condition 

164 unbalance_matches = ( 

165 unbalance_values is None or item.get("Unbalance") in unbalance_values 

166 ) 

167 

168 # Check for element count condition 

169 if element_key is None: 

170 element_matches = True 

171 else: 

172 element_count = item.get(formula_key, {}).get(element_key, 0) 

173 element_matches = min_count <= element_count <= max_count 

174 

175 if unbalance_matches and element_matches: 

176 filtered_data.append(item) 

177 

178 return filtered_data 

179 

180 

181def remove_duplicates_by_key( 

182 data: List[dict], key_function: Callable[..., Any] 

183) -> List[dict]: 

184 """ 

185 Remove duplicate entries from a list based on a unique key for each entry. 

186 

187 Parameters: 

188 - `data` (List[dict]): A list of data entries (dictionaries, objects, etc.). 

189 - `key_function` (Callable[..., Any]): A function that takes an entry from `data` and returns a key for duplicate check. 

190 

191 Returns: 

192 - `List[dict]`: A list of unique entries, based on the unique keys generated. 

193 

194 Example: 

195 >>> data = [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}, {'name': 'Alice', 'age': 30}] 

196 >>> remove_duplicates_by_key(data, lambda x: (x['name'], x['age'])) 

197 [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}] 

198 """ 

199 

200 seen_keys = set() 

201 unique_data = [] 

202 

203 for entry in data: 

204 key = frozenset(key_function(entry)) 

205 if key not in seen_keys: 

206 seen_keys.add(key) 

207 unique_data.append(entry) 

208 

209 return unique_data 

210 

211 

212def sort_by_key_length( 

213 data: List[Any], key_function: Callable[[Any], Any] 

214) -> List[Any]: 

215 """ 

216 Sort a list of entries based on the length of a specific key. 

217 

218 Args: 

219 - data (List[Any]): A list of data entries. 

220 - key_function (Callable[[Any], Any]): A function that takes an entry from `data` and returns a key 

221 whose length is to be used for sorting. 

222 

223 Returns: 

224 - List[Any]: A list of entries sorted by the length of the specified key. 

225 """ 

226 

227 return sorted(data, key=lambda x: len(key_function(x))) 

228 

229 

230def add_missing_key_to_dicts( 

231 data: List[Dict[str, Dict[str, Any]]], 

232 dict_key: str, 

233 missing_key: str, 

234 default_value: Any, 

235) -> List[Dict[str, Dict[str, Any]]]: 

236 """ 

237 Iterates through a list of dictionaries and adds a specified key with a default value to a specified 

238 dictionary within each main dictionary, if the key is not already present. Returns a new list with the updates. 

239 

240 Args: 

241 data: A list of dictionaries. 

242 dict_key: The key in the main dictionaries that points to another dictionary where the check should be done. 

243 missing_key: The key to add if it's not present in the nested dictionary. 

244 default_value: The default value to assign to the missing key. 

245 

246 Returns: 

247 A new list of dictionaries with the missing key added where necessary. 

248 

249 Example: 

250 >>> data = [{'Composition': {'A': 1, 'B': 2}}, {'Composition': {'B': 3}}] 

251 >>> updated_data = add_missing_key_to_dicts(data, 'Composition', 'Q', 0) 

252 >>> updated_data 

253 [{'Composition': {'A': 1, 'B': 2, 'Q': 0}}, {'Composition': {'B': 3, 'Q': 0}}] 

254 """ 

255 

256 updated_data = [] 

257 

258 for entry in data: 

259 # Create a copy of the entry to avoid modifying the original data 

260 updated_entry = entry.copy() 

261 

262 # Check if the missing key is not in the specified dictionary 

263 if missing_key not in updated_entry.get(dict_key, {}): 

264 # Ensure the dict_key points to a dictionary 

265 if dict_key not in updated_entry: 

266 updated_entry[dict_key] = {} 

267 

268 # Add the missing key with the default value 

269 updated_entry[dict_key][missing_key] = default_value 

270 

271 updated_data.append(updated_entry) 

272 

273 return updated_data 

274 

275 

276def extract_results_by_key( 

277 data: List[Dict[str, any]], key: str = "new_reaction" 

278) -> Tuple[List[Dict[str, any]], List[Dict[str, any]]]: 

279 """ 

280 Separate dictionaries from a list into two lists based on the presence of a specific key. 

281 

282 Args: 

283 data: A list of dictionaries to be separated. 

284 key: The key to check for in each dictionary. Defaults to 'new_reaction'. 

285 

286 Returns: 

287 A tuple of two lists: 

288 - The first list contains dictionaries that have the specified key. 

289 - The second list contains dictionaries that do not have the specified key. 

290 """ 

291 

292 with_key: List[Dict[str, any]] = [] 

293 without_key: List[Dict[str, any]] = [] 

294 

295 # Separate dictionaries based on the presence of the key 

296 for item in data: 

297 if key in item: 

298 with_key.append(item) 

299 else: 

300 without_key.append(item) 

301 

302 return with_key, without_key 

303 

304 

305def get_random_samples_by_key( 

306 data: List[Dict[str, Any]], 

307 stratify_key: str, 

308 num_samples_per_group: int = 1, 

309 random_seed: int = None, 

310) -> List[Dict[str, Any]]: 

311 """ 

312 Get random samples from data, grouped by a specified key. 

313 

314 Parameters: 

315 - data: List of dictionaries containing various keys. 

316 - stratify_key: The key used for stratifying the data. 

317 - num_samples_per_group: Number of random samples to draw from each unique group. 

318 - random_seed: Seed for the random number generator for reproducibility. 

319 

320 Returns: 

321 - A list of randomly selected samples from each unique group based on the stratify_key. 

322 """ 

323 if random_seed is not None: 

324 random.seed(random_seed) 

325 

326 # Function to create a sortable key from the item 

327 def sortable_key(item: Dict[str, Any]) -> Union[str, tuple]: 

328 key_value = item.get(stratify_key, None) 

329 if isinstance(key_value, dict): 

330 return tuple(sorted(key_value.items())) 

331 elif isinstance(key_value, list): 

332 return tuple(sorted(key_value)) 

333 else: 

334 return key_value 

335 

336 # Group data by the specified stratify_key 

337 grouped_data = defaultdict(list) 

338 for item in data: 

339 grouped_key = sortable_key(item) 

340 grouped_data[grouped_key].append(item) 

341 

342 # Select random samples from each group 

343 random_samples = [] 

344 for _, items in grouped_data.items(): 

345 if len(items) >= num_samples_per_group: 

346 random_samples.extend(random.sample(items, num_samples_per_group)) 

347 else: 

348 # If there aren't enough items, take all available 

349 random_samples.extend(items) 

350 

351 return random_samples