1 /*
2    Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
3 
4    This program is free software; you can redistribute it and/or modify
5    it under the terms of the GNU General Public License, version 2.0,
6    as published by the Free Software Foundation.
7 
8    This program is also distributed with certain software (including
9    but not limited to OpenSSL) that is licensed under separate terms,
10    as designated in a particular file or component or in included license
11    documentation.  The authors of MySQL hereby grant you an additional
12    permission to link the program and your derivative works with the
13    separately licensed software that they have included with MySQL.
14 
15    This program is distributed in the hope that it will be useful,
16    but WITHOUT ANY WARRANTY; without even the implied warranty of
17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18    GNU General Public License, version 2.0, for more details.
19 
20    You should have received a copy of the GNU General Public License
21    along with this program; if not, write to the Free Software
22    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
23 */
24 
25 #ifndef NDBCNTR_H
26 #define NDBCNTR_H
27 
28 
29 #include <pc.hpp>
30 #include <SimulatedBlock.hpp>
31 #include <ndb_limits.h>
32 #include <signaldata/RedoStateRep.hpp>
33 #include <signaldata/StopReq.hpp>
34 #include <signaldata/ResumeReq.hpp>
35 #include <signaldata/DictTabInfo.hpp>
36 #include <signaldata/CntrStart.hpp>
37 #include <signaldata/CheckNodeGroups.hpp>
38 #include <signaldata/LocalSysfile.hpp>
39 
40 #include <NodeState.hpp>
41 #include <NdbTick.h>
42 
43 #define JAM_FILE_ID 457
44 
45 
46 #ifdef NDBCNTR_C
47 /*
48 2.1 GLOBAL SYMBOLS
49 ------------------
50 */
51 /*
52 2.2 LOCAL SYMBOLS
53 -----------------
54 */
55 #define ZNO_NDB_BLOCKS 6           /* ACC, DICT, DIH, LQH, TC, TUP         */
56 
57 #define ZNOT_AVAILABLE 913
58 
59 //------- OTHERS ---------------------------------------------
60 #define ZSTARTUP  1
61 #define ZSHUTDOWN 2
62 #define ZBLOCK_STTOR 3
63 
64 #define ZSIZE_NDB_BLOCKS_REC 16 /* MAX BLOCKS IN NDB                    */
65 #define ZSTART_PHASE_1 1
66 #define ZSTART_PHASE_2 2
67 #define ZSTART_PHASE_3 3
68 #define ZSTART_PHASE_4 4
69 #define ZSTART_PHASE_5 5
70 #define ZSTART_PHASE_6 6
71 #define ZSTART_PHASE_7 7
72 #define ZSTART_PHASE_8 8
73 #define ZSTART_PHASE_9 9
74 #define ZSTART_PHASE_END 255
75 #endif
76 
77 class Ndbcntr: public SimulatedBlock {
78 public:
79 // Records
80 
81 /* FSREADREQ FSWRITEREQ         */
82 /**
83  * 2.3 RECORDS AND FILESIZES
84  * ------------------------------------------------------------
85  */
86 
87   struct StartRecord {
StartRecordNdbcntr::StartRecord88     StartRecord() {}
89     NDB_TICKS m_startTime;
90 
91     void reset();
92     NdbNodeBitmask m_starting;
93     NdbNodeBitmask m_waiting;
94     // == (m_withLog | m_withoutLog | m_withLogNotRestorable | m_waitTO)
95     NdbNodeBitmask m_withLog;
96     NdbNodeBitmask m_withLogNotRestorable;
97     NdbNodeBitmask m_withoutLog;
98     NdbNodeBitmask m_waitTO;
99     Uint32 m_lastGci;
100     Uint32 m_lastGciNodeId;
101     Uint32 m_lastLcpId;
102 
103     // Timeouts in ms since 'm_startTime'
104     Uint64 m_startPartialTimeout;  // UNUSED!
105     Uint64 m_startPartitionedTimeout;
106     Uint64 m_startFailureTimeout;
107     struct {
108       Uint32 m_nodeId;
109       Uint32 m_lastGci;
110     } m_logNodes[MAX_NDB_NODES];
111     Uint32 m_logNodesCount;
112 
113     Uint32 m_wait_sp[MAX_NDB_NODES];
114   } c_start;
115 
116   struct LocalSysfile
117   {
LocalSysfileNdbcntr::LocalSysfile118     LocalSysfile() {}
119     Uint32 m_data[128];
120     Uint32 m_file_pointer;
121     Uint32 m_sender_data;
122     Uint32 m_sender_ref;
123     bool m_initial_read_done;
124     bool m_last_write_done;
125     bool m_initial_write_local_sysfile_ongoing;
126     enum
127     {
128       NOT_USED = 0,
129       OPEN_READ_FILE_0 = 1,
130       OPEN_READ_FILE_1 = 2,
131       READ_FILE_0 = 3,
132       READ_FILE_1 = 4,
133       CLOSE_READ_FILE = 5,
134       CLOSE_READ_REF_0 = 6,
135       CLOSE_READ_REF_1 = 7,
136       OPEN_WRITE_FILE_0 = 8,
137       OPEN_WRITE_FILE_1 = 9,
138       WRITE_FILE_0 = 10,
139       WRITE_FILE_1 = 11,
140       CLOSE_WRITE_FILE_0 = 12,
141       CLOSE_WRITE_FILE_1 = 13
142     } m_state;
143     Uint32 m_restorable_flag;
144     Uint32 m_max_restorable_gci;
145   } c_local_sysfile;
146 
147   struct NdbBlocksRec {
148     BlockReference blockref;
149   }; /* p2c: size = 2 bytes */
150 
151   typedef Ptr<NdbBlocksRec> NdbBlocksRecPtr;
152 
153   /**
154    * Ndbcntr creates and initializes system tables on initial system start.
155    * The tables are defined in static structs in NdbcntrSysTable.cpp.
156    */
157   struct SysColumn {
158     unsigned pos;
159     const char* name;
160     // DictTabInfo
161     DictTabInfo::ExtType type;
162     Uint32 length;
163     bool keyFlag;
164     bool nullable;
165   };
166   struct SysTable {
167     const char* name;
168     unsigned columnCount;
169     const SysColumn* columnList;
170     // DictTabInfo
171     DictTabInfo::TableType tableType;
172     DictTabInfo::FragmentType fragmentType;
173     bool tableLoggedFlag;
174     // saved table id
175     mutable Uint32 tableId;
176     mutable Uint32 tableVersion;
177   };
178   struct SysIndex {
179     const char* name;
180     const SysTable* primaryTable;
181     Uint32 columnCount;
182     Uint32 columnList[4];
183     // DictTabInfo
184     DictTabInfo::TableType indexType;
185     DictTabInfo::FragmentType fragmentType;
186     bool indexLoggedFlag;
187     // saved index table id
188     mutable Uint32 indexId;
189   };
190   static const SysTable* g_sysTableList[];
191   static const unsigned g_sysTableCount;
192   // the system tables
193   static const SysTable g_sysTable_SYSTAB_0;
194   static SysTable g_sysTable_NDBEVENTS_0;
195   // schema trans
196   Uint32 c_schemaTransId;
197   Uint32 c_schemaTransKey;
198   // intersignal transient store of: hash_map, logfilegroup, tablesspace
199   Uint32 c_objectId;
200   Uint32 c_objectVersion;
201 
202 public:
203   Ndbcntr(Block_context&);
204   virtual ~Ndbcntr();
205 
206 private:
207   BLOCK_DEFINES(Ndbcntr);
208 
209   // Transit signals
210   void execAPI_START_REP(Signal*);
211   void execCONTINUEB(Signal* signal);
212   void execREAD_NODESCONF(Signal* signal);
213   void execREAD_NODESREF(Signal* signal);
214   void execCM_ADD_REP(Signal* signal);
215   void execCNTR_START_REQ(Signal* signal);
216   void execCNTR_START_REF(Signal* signal);
217   void execCNTR_START_CONF(Signal* signal);
218   void execCNTR_START_REP(Signal* signal);
219   void execCNTR_WAITREP(Signal* signal);
220   void execNODE_FAILREP(Signal* signal);
221   void execSYSTEM_ERROR(Signal* signal);
222 
223   void execSTART_PERMREP(Signal*);
224 
225   // Received signals
226   void execDUMP_STATE_ORD(Signal* signal);
227   void execREAD_CONFIG_REQ(Signal* signal);
228   void execSTTOR(Signal* signal);
229   void execGETGCICONF(Signal* signal);
230   void execDIH_RESTARTCONF(Signal* signal);
231   void execDIH_RESTARTREF(Signal* signal);
232   void execSET_UP_MULTI_TRP_CONF(Signal*);
233   void execSCHEMA_TRANS_BEGIN_CONF(Signal* signal);
234   void execSCHEMA_TRANS_BEGIN_REF(Signal* signal);
235   void execSCHEMA_TRANS_END_CONF(Signal* signal);
236   void execSCHEMA_TRANS_END_REF(Signal* signal);
237   void execCREATE_TABLE_REF(Signal* signal);
238   void execCREATE_TABLE_CONF(Signal* signal);
239   void execCREATE_HASH_MAP_REF(Signal* signal);
240   void execCREATE_HASH_MAP_CONF(Signal* signal);
241   void execCREATE_FILEGROUP_REF(Signal* signal);
242   void execCREATE_FILEGROUP_CONF(Signal* signal);
243   void execCREATE_FILE_REF(Signal* signal);
244   void execCREATE_FILE_CONF(Signal* signal);
245   void execNDB_STTORRY(Signal* signal);
246   void execNDB_STARTCONF(Signal* signal);
247   void execREAD_NODESREQ(Signal* signal);
248   void execNDB_STARTREF(Signal* signal);
249 
250   void execSTOP_PERM_REF(Signal* signal);
251   void execSTOP_PERM_CONF(Signal* signal);
252 
253   void execSTOP_ME_REF(Signal* signal);
254   void execSTOP_ME_CONF(Signal* signal);
255 
256   void execWAIT_GCP_REF(Signal* signal);
257   void execWAIT_GCP_CONF(Signal* signal);
258 
259   void execREDO_STATE_REP(Signal* signal);
260 
261   void execSTOP_REQ(Signal* signal);
262   void execSTOP_CONF(Signal* signal);
263   void execRESUME_REQ(Signal* signal);
264 
265   void execCHANGE_NODE_STATE_CONF(Signal* signal);
266 
267   void execABORT_ALL_REF(Signal* signal);
268   void execABORT_ALL_CONF(Signal* signal);
269 
270   // Statement blocks
271   void beginSchemaTransLab(Signal* signal);
272   void endSchemaTransLab(Signal* signal);
273   void sendCreateTabReq(Signal* signal, const char* buffer, Uint32 bufLen);
274   void initData(Signal* signal);
275   void resetStartVariables(Signal* signal);
276   void sendCntrStartReq(Signal* signal);
277   void sendCntrStartRef(Signal*, Uint32 nodeId, CntrStartRef::ErrorCode);
278   void sendNdbSttor(Signal* signal);
279   void sendSttorry(Signal* signal, Uint32 delayed = 0);
280 
281   bool trySystemRestart(Signal* signal);
282   void startWaitingNodes(Signal* signal);
283   CheckNodeGroups::Output checkNodeGroups(Signal*, const NdbNodeBitmask &);
284 
285   // Generated statement blocks
286   [[noreturn]] void systemErrorLab(Signal* signal, int line);
287 
288   void createHashMap(Signal*, Uint32 index);
289   void createSystableLab(Signal* signal, unsigned index);
290   void createDDObjects(Signal*, unsigned index);
291 
292   void startPhase1Lab(Signal* signal);
293   void startPhase2Lab(Signal* signal);
294   void startPhase3Lab(Signal* signal);
295   void startPhase4Lab(Signal* signal);
296   void startPhase5Lab(Signal* signal);
297   // jump 2 to resync phase counters
298   void startPhase8Lab(Signal* signal);
299   void startPhase9Lab(Signal* signal);
300   void ph2ALab(Signal* signal);
301   void ph2CLab(Signal* signal);
302   void ph2ELab(Signal* signal);
303   void ph2FLab(Signal* signal);
304   void ph2GLab(Signal* signal);
305   void ph3ALab(Signal* signal);
306   void ph4ALab(Signal* signal);
307   void ph4BLab(Signal* signal);
308   void ph4CLab(Signal* signal);
309   void ph5ALab(Signal* signal);
310   void ph6ALab(Signal* signal);
311   void ph6BLab(Signal* signal);
312   void ph7ALab(Signal* signal);
313   void ph8ALab(Signal* signal);
314 
315   void waitpoint41Lab(Signal* signal);
316   void waitpoint51Lab(Signal* signal);
317   void waitpoint52Lab(Signal* signal);
318   void waitpoint61Lab(Signal* signal);
319   void waitpoint71Lab(Signal* signal);
320   void waitpoint42To(Signal* signal);
321 
322   /**
323    * Wait before starting sp
324    *   so that all nodes in cluster is waiting for >= sp
325    */
326   bool wait_sp(Signal*, Uint32 sp);
327   void wait_sp_rep(Signal*);
328 
329   void execSTART_COPYREF(Signal*);
330   void execSTART_COPYCONF(Signal*);
331 
332   void execCREATE_NODEGROUP_IMPL_REQ(Signal*);
333   void execDROP_NODEGROUP_IMPL_REQ(Signal*);
334 
335   /* Local Sysfile stuff */
336   void execREAD_LOCAL_SYSFILE_REQ(Signal*);
337   void execWRITE_LOCAL_SYSFILE_REQ(Signal*);
338   void execREAD_LOCAL_SYSFILE_CONF(Signal*);
339   void execWRITE_LOCAL_SYSFILE_CONF(Signal*);
340   void execFSOPENREF(Signal*);
341   void execFSOPENCONF(Signal*);
342   void execFSREADREF(Signal*);
343   void execFSREADCONF(Signal*);
344   void execFSWRITECONF(Signal*);
345   void execFSWRITEREF(Signal*);
346   void execFSCLOSEREF(Signal*);
347   void execFSCLOSECONF(Signal*);
348 
349   void sendReadLocalSysfile(Signal *signal);
350   void sendWriteLocalSysfile_initial(Signal *signal);
351   void update_withLog();
352 
353   void init_local_sysfile();
354   void init_local_sysfile_vars();
355   void open_local_sysfile(Signal*, Uint32, bool);
356   void read_local_sysfile(Signal*);
357   void read_local_sysfile_data(Signal*);
358   void write_local_sysfile(Signal*);
359   void handle_read_refuse(Signal*);
360   void close_local_sysfile(Signal*);
361   void sendReadLocalSysfileConf(Signal*, BlockReference, Uint32);
362   void sendWriteLocalSysfileConf(Signal*);
363 
364   void updateNodeState(Signal* signal, const NodeState & newState) const ;
365   void getNodeGroup(Signal* signal);
366 
367   void send_node_started_rep(Signal *signal);
368 
369   // Initialisation
370   void initData();
371   void initRecords();
372 
373   // Variables
374   /**------------------------------------------------------------------------
375    * CONTAIN INFO ABOUT ALL NODES IN CLUSTER. NODE_PTR ARE USED AS NODE NUMBER
376    * IF THE STATE ARE ZDELETE THEN THE NODE DOESN'T EXIST. NODES ARE ALLOWED
377    * TO REGISTER (ZADD) DURING RESTART.
378    *------------------------------------------------------------------------*/
379   NdbBlocksRec *ndbBlocksRec;
380 
381   /*
382     2.4 COMMON STORED VARIABLES
383   */
384   UintR cnoWaitrep6;
385   UintR cnoWaitrep7;
386   UintR ctcReqInfo;
387 
388   Uint8 cstartPhase;
389   Uint16 cinternalStartphase;
390 
391   bool m_cntr_start_conf;
392   Uint16 cmasterNodeId;
393   Uint16 cndbBlocksCount;
394   Uint16 cnoStartNodes;
395   UintR cnoWaitrep;
396   NodeState::StartType ctypeOfStart;
397   NodeState::StartType cdihStartType;
398   Uint16 cdynamicNodeId;
399 
400   Uint32 c_fsRemoveCount;
401   Uint32 c_nodeGroup;
402   void clearFilesystem(Signal* signal);
403   void execFSREMOVECONF(Signal* signal);
404 
405   NdbNodeBitmask c_allDefinedNodes;
406   NdbNodeBitmask c_clusterNodes; // All members of qmgr cluster
407   /**
408    * c_cntr_startedNodeSet contains the nodes that have been allowed
409    * to start in CNTR_START_CONF. This is establised in phase 2 of the
410    * start.
411    *
412    * c_startedNodeSet contains the nodes that have completed the
413    * start and passed all start phases.
414    */
415   NdbNodeBitmask c_cntr_startedNodeSet;
416   NdbNodeBitmask c_startedNodeSet;
417 
418 public:
419   struct StopRecord {
420   public:
StopRecordNdbcntr::StopRecord421     StopRecord(Ndbcntr & _cntr) : cntr(_cntr) {
422       stopReq.senderRef = 0;
423     }
424 
425     Ndbcntr & cntr;
426     StopReq stopReq;          // Signal data
427     NDB_TICKS stopInitiatedTime; // When was the stop initiated
428 
429     bool checkNodeFail(Signal* signal);
430     void checkTimeout(Signal* signal);
431     void checkApiTimeout(Signal* signal);
432     void checkTcTimeout(Signal* signal);
433     void checkLqhTimeout_1(Signal* signal);
434     void checkLqhTimeout_2(Signal* signal);
435 
numberNdbcntr::StopRecord436     BlockNumber number() const { return cntr.number(); }
jamBufferNdbcntr::StopRecord437     EmulatedJamBuffer *jamBuffer() const { return cntr.jamBuffer(); }
progErrorNdbcntr::StopRecord438     [[noreturn]] void progError(int line,
439                                 int cause,
440                                 const char * extra,
441                                 const char * check)
442     {
443       cntr.progError(line, cause, extra, check);
444     }
445 
446     enum StopNodesStep {
447       SR_BLOCK_GCP_START_GCP = 0,
448       SR_WAIT_COMPLETE_GCP = 1,
449       SR_UNBLOCK_GCP_START_GCP = 2,
450       SR_QMGR_STOP_REQ = 3,
451       SR_WAIT_NODE_FAILURES = 4,
452       SR_CLUSTER_SHUTDOWN = 12
453     } m_state;
454     SignalCounter m_stop_req_counter;
455   };
456   bool is_node_started(NodeId);
457   bool is_node_starting(NodeId);
458 private:
459   bool is_nodegroup_starting(Signal*, NodeId);
460   void get_node_group_mask(Signal*, NodeId, NdbNodeBitmask&);
461 
462   StopRecord c_stopRec;
463   friend struct StopRecord;
464 
465   struct Missra {
MissraNdbcntr::Missra466     Missra(Ndbcntr & ref) : cntr(ref) { }
467 
468     Uint32 currentBlockIndex;
469     Uint32 currentStartPhase;
470     Uint32 nextStartPhase[NO_OF_BLOCKS];
471 
472     void execSTART_ORD(Signal* signal);
473     void execSTTORRY(Signal* signal);
474     void sendNextSTTOR(Signal* signal);
475     void execREAD_CONFIG_CONF(Signal* signal);
476     void sendNextREAD_CONFIG_REQ(Signal* signal);
477 
numberNdbcntr::Missra478     BlockNumber number() const { return cntr.number(); }
jamBufferNdbcntr::Missra479     EmulatedJamBuffer *jamBuffer() const { return cntr.jamBuffer(); }
progErrorNdbcntr::Missra480     [[noreturn]] void progError(int line,
481                                 int cause,
482                                 const char * extra,
483                                 const char * check)
484     {
485       cntr.progError(line, cause, extra, check);
486     }
487     Ndbcntr & cntr;
488   };
489 
490   Missra c_missra;
491   friend struct Missra;
492 
493   void execSTTORRY(Signal* signal);
494   void execSTART_ORD(Signal* signal);
495   void execREAD_CONFIG_CONF(Signal*);
496 
497   void send_restorable_gci_rep_to_backup(Signal*, Uint32);
498 
499   bool m_received_wait_all;
500   bool m_any_lcp_started;
501   bool m_initial_local_lcp_started;
502   bool m_local_lcp_started;
503   bool m_local_lcp_completed;
504   bool m_full_local_lcp_started;
505   bool m_distributed_lcp_started;
506   bool m_first_distributed_lcp_started;
507   bool m_ready_to_cut_log_tail;
508   bool m_wait_cut_undo_log_tail;
509   bool m_copy_fragment_in_progress;
510   Uint32 m_distributed_lcp_id;
511   Uint32 m_set_local_lcp_id_reqs;
512   Uint32 m_outstanding_wait_lcp;
513   Uint32 m_outstanding_wait_cut_redo_log_tail;
514   Uint32 m_max_gci_in_lcp;
515   Uint32 m_max_keep_gci;
516   Uint32 m_max_completed_gci;
517 
518   Uint32 m_lcp_id;
519   Uint32 m_local_lcp_id;
520   RedoStateRep::RedoAlertState m_global_redo_alert_state;
521   RedoStateRep::RedoAlertState m_node_redo_alert_state;
522   RedoStateRep::RedoAlertState m_redo_alert_state[MAX_NDBMT_LQH_THREADS];
523 
524   RedoStateRep::RedoAlertState get_node_redo_alert_state();
525   Uint32 send_to_all_lqh(Signal*, Uint32 gsn, Uint32 sig_len);
526   Uint32 send_to_all_backup(Signal*, Uint32 gsn, Uint32 sig_len);
527   void send_cut_log_tail(Signal*);
528   void check_cut_log_tail_completed(Signal*);
529   bool is_ready_to_cut_log_tail();
530   void sendWAIT_ALL_COMPLETE_LCP_CONF(Signal*);
531   void sendLCP_ALL_COMPLETE_CONF(Signal*);
532   void sendSTART_FULL_LOCAL_LCP_ORD(Signal*);
533   void sendSTART_LOCAL_LCP_ORD(Signal*);
534   void sendSET_LOCAL_LCP_ID_CONF(Signal*);
535   void sendWriteLocalSysfile_startLcp(Signal*,Uint32);
536   void write_local_sysfile_start_lcp_done(Signal*);
537   const char* get_restorable_flag_string(Uint32);
538 
539   void execCOPY_FRAG_IN_PROGRESS_REP(Signal*);
540   void execCOPY_FRAG_NOT_IN_PROGRESS_REP(Signal*);
541   void execUNDO_LOG_LEVEL_REP(Signal*);
542   void execSTART_LOCAL_LCP_ORD(Signal*);
543   void execSET_LOCAL_LCP_ID_REQ(Signal*);
544   void execWAIT_ALL_COMPLETE_LCP_REQ(Signal*);
545   void execWAIT_COMPLETE_LCP_CONF(Signal*);
546 
547   void execSTART_DISTRIBUTED_LCP_ORD(Signal*);
548   void execLCP_ALL_COMPLETE_REQ(Signal*);
549 
550   void execCUT_UNDO_LOG_TAIL_CONF(Signal*);
551   void execCUT_REDO_LOG_TAIL_CONF(Signal*);
552 
553   void execRESTORABLE_GCI_REP(Signal*);
554 };
555 
556 
557 #undef JAM_FILE_ID
558 
559 #endif
560