0006-Issue-51102-RFE-ds-replcheck-make-online-timeout-con.patch 11.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
From c0cb15445c1434b3d317b1c06ab1a0ba8dbc6f04 Mon Sep 17 00:00:00 2001
From: Mark Reynolds <mreynolds@redhat.com>
Date: Tue, 19 May 2020 15:11:53 -0400
Subject: [PATCH 06/12] Issue 51102 - RFE - ds-replcheck - make online timeout
 configurable

Bug Description:  When doing an online check with replicas that are very
                  far apart the connection can time out as the hardcoded
                  timeout is 5 seconds.

Fix Description:  Change the default timeout to never timeout, and add an
                  CLI option to specify a specific timeout.

                  Also caught all the possible LDAP exceptions so we can
                  cleanly "fail".  Fixed some python syntax issues, and
                  improved the entry inconsistency report

relates: https://pagure.io/389-ds-base/issue/51102

Reviewed by: firstyear & spichugi(Thanks!)
---
 ldap/admin/src/scripts/ds-replcheck | 90 ++++++++++++++++++-----------
 1 file changed, 57 insertions(+), 33 deletions(-)

diff --git a/ldap/admin/src/scripts/ds-replcheck b/ldap/admin/src/scripts/ds-replcheck
index 30bcfd65d..5bb7dfce3 100755
--- a/ldap/admin/src/scripts/ds-replcheck
+++ b/ldap/admin/src/scripts/ds-replcheck
@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
 # --- BEGIN COPYRIGHT BLOCK ---
-# Copyright (C) 2018 Red Hat, Inc.
+# Copyright (C) 2020 Red Hat, Inc.
 # All rights reserved.
 #
 # License: GPL (version 3 or any later version).
@@ -21,10 +21,9 @@ import getpass
 import signal
 from ldif import LDIFRecordList
 from ldap.ldapobject import SimpleLDAPObject
-from ldap.cidict import cidict
 from ldap.controls import SimplePagedResultsControl
 from lib389._entry import Entry
-from lib389.utils import ensure_str, ensure_list_str, ensure_int
+from lib389.utils import ensure_list_str, ensure_int
 
 VERSION = "2.0"
 RUV_FILTER = '(&(nsuniqueid=ffffffff-ffffffff-ffffffff-ffffffff)(objectclass=nstombstone))'
@@ -185,11 +184,11 @@ def report_conflict(entry, attr, opts):
     report = True
 
     if 'nscpentrywsi' in entry.data:
-        found = False
         for val in entry.data['nscpentrywsi']:
             if val.lower().startswith(attr + ';'):
                 if (opts['starttime'] - extract_time(val)) <= opts['lag']:
                     report = False
+                    break
 
     return report
 
@@ -321,6 +320,9 @@ def ldif_search(LDIF, dn):
     count = 0
     ignore_list = ['conflictcsn', 'modifytimestamp', 'modifiersname']
     val = ""
+    attr = ""
+    state_attr = ""
+    part_dn = ""
     result['entry'] = None
     result['conflict'] = None
     result['tombstone'] = False
@@ -570,6 +572,7 @@ def cmp_entry(mentry, rentry, opts):
                         if val.lower().startswith(mattr + ';'):
                             if not found:
                                 diff['diff'].append("      Master:")
+                            diff['diff'].append("        - Value:      %s" % (val.split(':')[1].lstrip()))
                             diff['diff'].append("        - State Info: %s" % (val))
                             diff['diff'].append("        - Date:       %s\n" % (time.ctime(extract_time(val))))
                             found = True
@@ -588,6 +591,7 @@ def cmp_entry(mentry, rentry, opts):
                         if val.lower().startswith(mattr + ';'):
                             if not found:
                                 diff['diff'].append("      Replica:")
+                            diff['diff'].append("        - Value:      %s" % (val.split(':')[1].lstrip()))
                             diff['diff'].append("        - State Info: %s" % (val))
                             diff['diff'].append("        - Date:       %s\n" % (time.ctime(extract_time(val))))
                             found = True
@@ -654,7 +658,6 @@ def do_offline_report(opts, output_file=None):
     rconflicts = []
     rtombstones = 0
     mtombstones = 0
-    idx = 0
 
     # Open LDIF files
     try:
@@ -926,7 +929,7 @@ def validate_suffix(ldapnode, suffix, hostname):
     :return - True if suffix exists, otherwise False
     """
     try:
-        master_basesuffix = ldapnode.search_s(suffix, ldap.SCOPE_BASE )
+        ldapnode.search_s(suffix, ldap.SCOPE_BASE)
     except ldap.NO_SUCH_OBJECT:
         print("Error: Failed to validate suffix in {}. {} does not exist.".format(hostname, suffix))
         return False
@@ -968,12 +971,12 @@ def connect_to_replicas(opts):
     replica = SimpleLDAPObject(ruri)
 
     # Set timeouts
-    master.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
-    master.set_option(ldap.OPT_TIMEOUT,5.0)
-    replica.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
-    replica.set_option(ldap.OPT_TIMEOUT,5.0)
+    master.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])
+    master.set_option(ldap.OPT_TIMEOUT, opts['timeout'])
+    replica.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])
+    replica.set_option(ldap.OPT_TIMEOUT, opts['timeout'])
 
-    # Setup Secure Conenction
+    # Setup Secure Connection
     if opts['certdir'] is not None:
         # Setup Master
         if opts['mprotocol'] != LDAPI:
@@ -1003,7 +1006,7 @@ def connect_to_replicas(opts):
     try:
         master.simple_bind_s(opts['binddn'], opts['bindpw'])
     except ldap.SERVER_DOWN as e:
-        print("Cannot connect to %r" % muri)
+        print(f"Cannot connect to {muri} ({str(e)})")
         sys.exit(1)
     except ldap.LDAPError as e:
         print("Error: Failed to authenticate to Master: ({}).  "
@@ -1014,7 +1017,7 @@ def connect_to_replicas(opts):
     try:
         replica.simple_bind_s(opts['binddn'], opts['bindpw'])
     except ldap.SERVER_DOWN as e:
-        print("Cannot connect to %r" % ruri)
+        print(f"Cannot connect to {ruri} ({str(e)})")
         sys.exit(1)
     except ldap.LDAPError as e:
         print("Error: Failed to authenticate to Replica: ({}).  "
@@ -1218,7 +1221,6 @@ def do_online_report(opts, output_file=None):
     """
     m_done = False
     r_done = False
-    done = False
     report = {}
     report['diff'] = []
     report['m_missing'] = []
@@ -1257,15 +1259,22 @@ def do_online_report(opts, output_file=None):
 
     # Read the results and start comparing
     while not m_done or not r_done:
-        if not m_done:
-            m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)
-        elif not r_done:
-            m_rdata = []
-
-        if not r_done:
-            r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)
-        elif not m_done:
-            r_rdata = []
+        try:
+            if not m_done:
+                m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)
+            elif not r_done:
+                m_rdata = []
+        except ldap.LDAPError as e:
+            print("Error: Problem getting the results from the master: %s", str(e))
+            sys.exit(1)
+        try:
+            if not r_done:
+                r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)
+            elif not m_done:
+                r_rdata = []
+        except ldap.LDAPError as e:
+            print("Error: Problem getting the results from the replica: %s", str(e))
+            sys.exit(1)
 
         # Convert entries
         mresult = convert_entries(m_rdata)
@@ -1291,11 +1300,15 @@ def do_online_report(opts, output_file=None):
                 ]
             if m_pctrls:
                 if m_pctrls[0].cookie:
-                    # Copy cookie from response control to request control
-                    req_pr_ctrl.cookie = m_pctrls[0].cookie
-                    master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
-                        "(|(objectclass=*)(objectclass=ldapsubentry))",
-                        ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+                    try:
+                        # Copy cookie from response control to request control
+                        req_pr_ctrl.cookie = m_pctrls[0].cookie
+                        master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
+                            "(|(objectclass=*)(objectclass=ldapsubentry))",
+                            ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+                    except ldap.LDAPError as e:
+                        print("Error: Problem searching the master: %s", str(e))
+                        sys.exit(1)
                 else:
                     m_done = True  # No more pages available
             else:
@@ -1311,11 +1324,15 @@ def do_online_report(opts, output_file=None):
 
             if r_pctrls:
                 if r_pctrls[0].cookie:
-                    # Copy cookie from response control to request control
-                    req_pr_ctrl.cookie = r_pctrls[0].cookie
-                    replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
-                        "(|(objectclass=*)(objectclass=ldapsubentry))",
-                        ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+                    try:
+                        # Copy cookie from response control to request control
+                        req_pr_ctrl.cookie = r_pctrls[0].cookie
+                        replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
+                            "(|(objectclass=*)(objectclass=ldapsubentry))",
+                            ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+                    except ldap.LDAPError as e:
+                        print("Error: Problem searching the replica: %s", str(e))
+                        sys.exit(1)
                 else:
                     r_done = True  # No more pages available
             else:
@@ -1426,6 +1443,9 @@ def init_online_params(args):
         # prompt for password
         opts['bindpw'] = getpass.getpass('Enter password: ')
 
+    # lastly handle the timeout
+    opts['timeout'] = int(args.timeout)
+
     return opts
 
 
@@ -1553,6 +1573,8 @@ def main():
     state_parser.add_argument('-y', '--pass-file', help='A text file containing the clear text password for the bind dn', dest='pass_file', default=None)
     state_parser.add_argument('-Z', '--cert-dir', help='The certificate database directory for secure connections',
                               dest='certdir', default=None)
+    state_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections.  Default is no timeout.',
+                              type=int, dest='timeout', default=-1)
 
     # Online mode
     online_parser = subparsers.add_parser('online', help="Compare two online replicas for differences")
@@ -1577,6 +1599,8 @@ def main():
     online_parser.add_argument('-p', '--page-size', help='The paged-search result grouping size (default 500 entries)',
                                dest='pagesize', default=500)
     online_parser.add_argument('-o', '--out-file', help='The output file', dest='file', default=None)
+    online_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections.  Default is no timeout.',
+                               type=int, dest='timeout', default=-1)
 
     # Offline LDIF mode
     offline_parser = subparsers.add_parser('offline', help="Compare two replication LDIF files for differences (LDIF file generated by 'db2ldif -r')")
-- 
2.26.2