1 #ifndef NETSCHEDULE_DB_DUMP__HPP 2 #define NETSCHEDULE_DB_DUMP__HPP 3 4 /* $Id: ns_db_dump.hpp 601572 2020-02-10 18:59:01Z satskyse $ 5 * =========================================================================== 6 * 7 * PUBLIC DOMAIN NOTICE 8 * National Center for Biotechnology Information 9 * 10 * This software/database is a "United States Government Work" under the 11 * terms of the United States Copyright Act. It was written as part of 12 * the author's official duties as a United States Government employee and 13 * thus cannot be copyrighted. This software/database is freely available 14 * to the public for use. The National Library of Medicine and the U.S. 15 * Government have not placed any restriction on its use or reproduction. 16 * 17 * Although all reasonable efforts have been taken to ensure the accuracy 18 * and reliability of the software and data, the NLM and the U.S. 19 * Government do not and cannot warrant the performance or results that 20 * may be obtained by using this software or data. The NLM and the U.S. 21 * Government disclaim all warranties, express or implied, including 22 * warranties of performance, merchantability or fitness for any particular 23 * purpose. 24 * 25 * Please cite the author in any work or product based on this material. 26 * 27 * =========================================================================== 28 * 29 * Authors: Sergey Satskiy 30 * 31 * File Description: 32 * NetSchedule database dumping support. 33 * 34 */ 35 36 #include "ns_types.hpp" 37 38 #include <corelib/ncbitype.h> 39 40 #include <stdio.h> 41 42 43 // NB 44 // The dump format is extendable and the extensions will work in both 45 // directions: forward and backward. The only requirement to keep it working is 46 // to extend the structures at the end without touching the existing part. 47 // In this case: 48 // - if a newer format is provided the older format dump (new NS starts after 49 // old NS saved the data) then: the extended fields will have zeroes. 50 // - if an older format is provided the newer data (old NS starts after a new 51 // NS, e.g. due to a rollback) then: the extended values are stripped. 52 53 54 55 BEGIN_NCBI_SCOPE 56 57 58 // Common header for the dump files 59 #pragma pack(push, 1) 60 struct SCommonDumpHeader 61 { 62 Int4 magic; 63 Uint4 storage_ver_major; 64 Uint4 storage_ver_minor; 65 Uint4 storage_ver_patch; 66 67 Uint4 server_ver_major; 68 Uint4 server_ver_minor; 69 Uint4 server_ver_patch; 70 71 SCommonDumpHeader(); 72 }; 73 #pragma pack(pop) 74 75 76 // The header appears ones in the job info file 77 #pragma pack(push, 1) 78 struct SJobDumpHeader 79 { 80 SCommonDumpHeader common_header; 81 Uint4 job_props_fixed_size; 82 Uint4 job_io_fixed_size; 83 Uint4 job_event_fixed_size; 84 85 SJobDumpHeader(); 86 87 void Write(FILE * f); 88 int Read(FILE * f); 89 }; 90 #pragma pack(pop) 91 92 93 // A few dump files store a sequence of one structure. They will use the header 94 // below 95 #pragma pack(push, 1) 96 struct SOneStructDumpHeader 97 { 98 SCommonDumpHeader common_header; 99 Uint4 fixed_size; 100 101 SOneStructDumpHeader(); 102 103 void Write(FILE * f); 104 int Read(FILE * f); 105 }; 106 #pragma pack(pop) 107 108 109 // The structures mostly match the Berkeley DB tables structure 110 // but use only fixed size POD fields 111 112 // It is supposed that the progress message follows each instance of the 113 // structure below. The progress message is the largest (and rare used) field 114 // in the structure and it affects the dumping performance and disk usage. 115 #pragma pack(push, 1) 116 struct SJobDump 117 { 118 Uint4 total_size; 119 Uint4 id; 120 121 Uint4 passport; 122 Int4 status; 123 double timeout; 124 double run_timeout; 125 double read_timeout; 126 127 Uint4 subm_notif_port; 128 double subm_notif_timeout; 129 130 Uint4 listener_notif_addr; 131 Uint4 listener_notif_port; 132 double listener_notif_abstime; 133 bool need_subm_progress_msg_notif; 134 bool need_lsnr_progress_msg_notif; 135 bool need_stolen_notif; 136 137 Uint4 run_counter; 138 Uint4 read_counter; 139 140 Uint4 aff_id; 141 Uint4 mask; 142 143 Uint4 group_id; 144 145 double last_touch; 146 Uint4 progress_msg_size; 147 Uint4 number_of_events; 148 149 Uint4 client_ip_size; 150 Uint4 client_sid_size; 151 Uint4 ncbi_phid_size; 152 char client_ip[kMaxClientIpSize]; 153 char client_sid[kMaxSessionIdSize]; 154 char ncbi_phid[kMaxHitIdSize]; 155 156 SJobDump(); 157 158 void Write(FILE * f, const char * progress_msg); 159 int Read(FILE * f, size_t fixed_size_from_header, 160 char * progress_msg); 161 }; 162 #pragma pack(pop) 163 164 165 // It is supposed that the input and/or output follow each instance of the 166 // structure below. The limit of the input/output is too large to have 167 // it as a member of this structure 168 #pragma pack(push, 1) 169 struct SJobIODump 170 { 171 Uint4 total_size; 172 Uint4 input_size; 173 Uint4 output_size; 174 175 SJobIODump(); 176 177 void Write(FILE * f, const char * input, const char * output); 178 int Read(FILE * f, size_t fixed_size_from_header, 179 char * input, char * output); 180 }; 181 #pragma pack(pop) 182 183 184 // It is supposed that the client_node, client_session and err_msg follow 185 // each instance of the structure below. There are a lot of the job events 186 // and otherwise it leads to both dumping speed and disk usage problems. 187 #pragma pack(push, 1) 188 struct SJobEventsDump 189 { 190 Uint4 total_size; 191 Uint4 event; 192 Int4 status; 193 double timestamp; 194 Uint4 node_addr; 195 Int4 ret_code; 196 Uint4 client_node_size; 197 Uint4 client_session_size; 198 Uint4 err_msg_size; 199 200 SJobEventsDump(); 201 202 void Write(FILE * f, const char * client_node, 203 const char * client_session, 204 const char * err_msg); 205 int Read(FILE * f, size_t fixed_size_from_header, 206 char * client_node, 207 char * client_session, 208 char * err_msg); 209 }; 210 #pragma pack(pop) 211 212 213 #pragma pack(push, 1) 214 struct SAffinityDictDump 215 { 216 Uint4 total_size; 217 Uint4 aff_id; 218 Uint4 token_size; 219 char token[kNetScheduleMaxDBDataSize]; 220 221 SAffinityDictDump(); 222 223 void Write(FILE * f); 224 int Read(FILE * f, size_t fixed_size_from_header); 225 }; 226 #pragma pack(pop) 227 228 229 #pragma pack(push, 1) 230 struct SGroupDictDump 231 { 232 Uint4 total_size; 233 Uint4 group_id; 234 Uint4 token_size; 235 char token[kNetScheduleMaxDBDataSize]; 236 237 SGroupDictDump(); 238 239 void Write(FILE * f); 240 int Read(FILE * f, size_t fixed_size_from_header); 241 }; 242 #pragma pack(pop) 243 244 245 #pragma pack(push, 1) 246 struct SQueueDescriptionDump 247 { 248 Uint4 total_size; 249 bool is_queue; 250 Uint4 qname_size; 251 char qname[kMaxQueueNameSize]; 252 Uint4 qclass_size; 253 char qclass[kMaxQueueNameSize]; 254 double timeout; 255 double notif_hifreq_interval; 256 double notif_hifreq_period; 257 Uint4 notif_lofreq_mult; 258 double notif_handicap; 259 Uint4 dump_buffer_size; 260 Uint4 dump_client_buffer_size; 261 Uint4 dump_aff_buffer_size; 262 Uint4 dump_group_buffer_size; 263 double run_timeout; 264 double read_timeout; 265 Uint4 program_name_size; 266 char program_name[kMaxQueueLimitsSize]; 267 Uint4 failed_retries; 268 Uint4 read_failed_retries; 269 double blacklist_time; 270 double read_blacklist_time; 271 Uint4 max_input_size; 272 Uint4 max_output_size; 273 Uint4 subm_hosts_size; 274 char subm_hosts[kMaxQueueLimitsSize]; 275 Uint4 wnode_hosts_size; 276 char wnode_hosts[kMaxQueueLimitsSize]; 277 Uint4 reader_hosts_size; 278 char reader_hosts[kMaxQueueLimitsSize]; 279 double wnode_timeout; 280 double reader_timeout; 281 double pending_timeout; 282 double max_pending_wait_timeout; 283 double max_pending_read_wait_timeout; 284 Uint4 description_size; 285 char description[kMaxDescriptionSize]; 286 bool scramble_job_keys; 287 double client_registry_timeout_worker_node; 288 Uint4 client_registry_min_worker_nodes; 289 double client_registry_timeout_admin; 290 Uint4 client_registry_min_admins; 291 double client_registry_timeout_submitter; 292 Uint4 client_registry_min_submitters; 293 double client_registry_timeout_reader; 294 Uint4 client_registry_min_readers; 295 double client_registry_timeout_unknown; 296 Uint4 client_registry_min_unknowns; 297 298 Uint4 linked_section_prefixes_size; 299 char linked_section_prefixes[kLinkedSectionsList]; 300 Uint4 linked_section_names_size; 301 char linked_section_names[kLinkedSectionsList]; 302 303 Uint4 max_jobs_per_client; 304 305 SQueueDescriptionDump(); 306 307 void Write(FILE * f); 308 int Read(FILE * f, size_t fixed_size_from_header); 309 }; 310 #pragma pack(pop) 311 312 313 #pragma pack(push, 1) 314 struct SLinkedSectionDump 315 { 316 Uint4 total_size; 317 Uint4 section_size; 318 Uint4 value_name_size; 319 Uint4 value_size; 320 char section[kLinkedSectionValueNameSize]; 321 char value_name[kLinkedSectionValueNameSize]; 322 char value[kLinkedSectionValueSize]; 323 324 SLinkedSectionDump(); 325 326 void Write(FILE * f); 327 int Read(FILE * f, size_t fixed_size_from_header); 328 }; 329 #pragma pack(pop) 330 331 332 END_NCBI_SCOPE 333 334 #endif /* NETSCHEDULE_DB_DUMP__HPP */ 335