68
68
#define GUARDIAN_FILE "guardian" /* name of the guardian file */
69
69
#define VERSION_FILE "DBVERSION" /* name of the version file */
70
#define MAX_TRIALS 50 /* number of retries on db operations */
71
70
#define V_5 5 /* changelog entry version */
72
71
#define CHUNK_SIZE 64*1024
73
72
#define DBID_SIZE 64
88
87
used to store upper boundary RUV vector */
90
89
#define DB_EXTENSION_DB3 "db3"
90
#define DB_EXTENSION_DB4 "db4"
91
#if 1000*DB_VERSION_MAJOR + 100*DB_VERSION_MINOR >= 5000
92
#define DB_EXTENSION "db"
91
94
#define DB_EXTENSION "db4"
93
97
#define HASH_BACKETS_COUNT 16 /* number of buckets in a hash table */
273
277
/* changelog initialization and cleanup */
274
278
static int _cl5Open (const char *dir, const CL5DBConfig *config, CL5OpenMode openMode);
275
static int _cl5AppInit (PRBool *didRecovery);
276
static int _cl5DBOpen ();
279
static int _cl5AppInit (void);
280
static int _cl5DBOpen (void);
277
281
static void _cl5SetDefaultDBConfig ();
278
282
static void _cl5SetDBConfig (const CL5DBConfig *config);
279
283
static int _cl5CheckDBVersion ();
323
327
static void _cl5WriteString (const char *str, char **buff);
324
328
static void _cl5ReadString (char **str, char **buff);
325
329
static void _cl5WriteMods (LDAPMod **mods, char **buff);
326
static void _cl5WriteMod (LDAPMod *mod, char **buff);
330
static int _cl5WriteMod (LDAPMod *mod, char **buff);
327
331
static int _cl5ReadMods (LDAPMod ***mods, char **buff);
328
332
static int _cl5ReadMod (Slapi_Mod *mod, char **buff);
329
333
static int _cl5GetModsSize (LDAPMod **mods);
331
335
static void _cl5ReadBerval (struct berval *bv, char** buff);
332
336
static void _cl5WriteBerval (struct berval *bv, char** buff);
333
337
static int _cl5ReadBervals (struct berval ***bv, char** buff, unsigned int size);
334
static int _cl5WriteBervals (struct berval **bv, char** buff, unsigned int *size);
338
static int _cl5WriteBervals (struct berval **bv, char** buff, u_int32_t *size);
336
340
/* replay iteration */
337
341
#ifdef FOR_DEBUGGING
877
881
#if defined(USE_OPENLDAP)
878
882
LDIFFP *file = NULL;
884
ldif_record_lineno_t lineno = 0;
881
886
FILE *file = NULL;
884
890
char *buff = NULL;
886
891
slapi_operation_parameters op;
887
892
Object *prim_replica_obj = NULL;
888
893
Object *replica_obj = NULL;
1899
1903
s_cl5Desc.dbOpenMode = openMode;
1901
1905
/* initialize db environment */
1902
rc = _cl5AppInit (&didRecovery);
1906
rc = _cl5AppInit ();
1903
1907
if (rc != CL5_SUCCESS)
1905
1909
slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl,
2029
2033
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
2030
2034
"_cl5AppInit: failed to fetch backend dbenv (%p) and/or "
2031
"index page size (%lu)\n", dbEnv, pagesize);
2035
"index page size (%lu)\n", dbEnv, (long unsigned int)pagesize);
2032
2036
return CL5_DB_ERROR;
2484
2488
mod_start = (*buff) + sizeof (count);
2486
2490
/* write mods*/
2487
for (i=0; mods[i]; i++)
2489
_cl5WriteMod (mods[i], &mod_start);
2491
for (i = 0; mods[i]; i++) {
2492
if (0 <= _cl5WriteMod (mods[i], &mod_start)) {
2492
count = PR_htonl(i);
2497
count = PR_htonl(count);
2493
2498
memcpy (*buff, &count, sizeof (count));
2495
2500
(*buff) = mod_start;
2498
static void _cl5WriteMod (LDAPMod *mod, char **buff)
2505
* positive: no need to encrypt && succeeded to write a mod
2506
* 0: succeeded to encrypt && write a mod
2507
* netative: failed to encrypt && no write to the changelog
2510
_cl5WriteMod (LDAPMod *mod, char **buff)
2502
2515
struct berval *bv;
2503
2516
struct berval *encbv;
2504
2517
struct berval *bv_to_use;
2505
2518
Slapi_Mod smod;
2524
if (SLAPD_UNHASHED_PW_NOLOG == slapi_config_get_unhashed_pw_switch()) {
2525
if (0 == strcasecmp(mod->mod_type, PSEUDO_ATTR_UNHASHEDUSERPASSWORD)) {
2526
/* If nsslapd-unhashed-pw-switch == nolog, skip writing it to cl. */
2508
2531
slapi_mod_init_byref(&smod, mod);
2533
orig_pos = pos = *buff;
2511
2534
/* write mod op */
2512
2535
*pos = (PRUint8)slapi_mod_get_operation (&smod);
2517
2540
/* write value count */
2518
2541
count = PR_htonl(slapi_mod_get_num_values(&smod));
2519
2542
memcpy (pos, &count, sizeof (count));
2520
pos += sizeof (PRInt32);
2543
pos += sizeof (PRInt32);
2545
/* if the mod has no values, eg delete attr or replace attr without values
2546
* do not reset buffer
2522
2550
bv = slapi_mod_get_first_value (&smod);
2527
2554
rc = clcrypt_encrypt_value(s_cl5Desc.clcrypt_handle,
2537
2564
"_cl5WriteMod: encrypting \"%s: %s\" failed\n",
2538
2565
slapi_mod_get_type(&smod), bv->bv_val);
2539
2566
bv_to_use = NULL;
2541
2570
if (bv_to_use) {
2542
2571
_cl5WriteBerval (bv_to_use, &pos);
2751
2785
PRUint32 net_length = 0;
2753
2787
length = (PRUint32) bv->bv_len;
2754
net_length = PR_htonl(length);
2788
net_length = PR_htonl(length);
2756
2790
memcpy(*buff, &net_length, sizeof (net_length));
2757
2791
*buff += sizeof (net_length);
2803
2837
/* data format: <value count> <value size> <value> <value size> <value> ..... */
2804
static int _cl5WriteBervals (struct berval **bv, char** buff, unsigned int *size)
2838
static int _cl5WriteBervals (struct berval **bv, char** buff, u_int32_t *size)
2806
2840
PRInt32 count, net_count;
2843
2877
* 3. Remove any Berkeley DB transaction log files
2844
2878
* 4. extention .db3 -> .db4
2846
static int _cl5Upgrade3_4(char *fromVersion, char *toVersion)
2880
static int _cl5UpgradeMajor(char *fromVersion, char *toVersion)
2848
2882
PRDir *dir = NULL;
2849
2883
PRDirEntry *entry = NULL;
2854
2888
backup = s_cl5Desc.dbOpenMode;
2855
2889
s_cl5Desc.dbOpenMode = CL5_OPEN_CLEAN_RECOVER;
2856
2890
/* CL5_OPEN_CLEAN_RECOVER does 1 and 2 */
2857
rc = _cl5AppInit (NULL);
2891
rc = _cl5AppInit ();
2858
2892
if (rc != CL5_SUCCESS)
2860
2894
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
2861
"_cl5Upgrade3_4: failed to open the db env\n");
2895
"_cl5UpgradeMajor: failed to open the db env\n");
2864
2898
s_cl5Desc.dbOpenMode = backup;
2867
2901
if (dir == NULL)
2869
2903
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
2870
"_cl5Upgrade3_4: failed to open changelog dir %s; NSPR error - %d\n",
2904
"_cl5UpgradeMajor: failed to open changelog dir %s; NSPR error - %d\n",
2871
2905
s_cl5Desc.dbDir, PR_GetError ());
2881
if (_cl5FileEndsWith(entry->name, DB_EXTENSION_DB3))
2915
if (_cl5FileEndsWith(entry->name, DB_EXTENSION_DB3) ||
2916
_cl5FileEndsWith(entry->name, DB_EXTENSION_DB4))
2883
2918
char oName [MAXPATHLEN + 1];
2884
2919
char nName [MAXPATHLEN + 1];
2889
2924
p = strstr(oName, DB_EXTENSION_DB3);
2927
p = strstr(oName, DB_EXTENSION_DB4);
2894
2933
/* db->rename closes DB; need to create every time */
2895
2934
rc = db_create(&thisdb, s_cl5Desc.dbEnv, 0);
2897
2936
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
2898
"_cl5Upgrade3_4: failed to get db handle\n");
2937
"_cl5UpgradeMajor: failed to get db handle\n");
2906
2945
PR_snprintf(nName + baselen, MAXPATHLEN+1-baselen, "%s", DB_EXTENSION);
2908
2947
slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl,
2909
"_cl5Upgrade3_4: renaming %s to %s\n", oName, nName);
2948
"_cl5UpgradeMajor: renaming %s to %s\n", oName, nName);
2910
2949
rc = thisdb->rename(thisdb, (const char *)oName, NULL /* subdb */,
2911
2950
(const char *)nName, 0);
2912
2951
if (rc != PR_SUCCESS)
2914
2953
slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl,
2915
"_cl5Upgrade3_4: failed to rename file (%s -> %s); "
2954
"_cl5UpgradeMajor: failed to rename file (%s -> %s); "
2916
2955
"db error - %d %s\n", oName, nName, rc, db_strerror(rc));
2936
2975
* 2. Remove any Berkeley DB environment using the DB_ENV->remove method
2937
2976
* 3. Remove any Berkeley DB transaction log files
2939
static int _cl5Upgrade4_4(char *fromVersion, char *toVersion)
2978
static int _cl5UpgradeMinor(char *fromVersion, char *toVersion)
2941
2980
CL5OpenMode backup;
2944
2983
backup = s_cl5Desc.dbOpenMode;
2945
2984
s_cl5Desc.dbOpenMode = CL5_OPEN_CLEAN_RECOVER;
2946
2985
/* CL5_OPEN_CLEAN_RECOVER does 1 and 2 */
2947
rc = _cl5AppInit (NULL);
2986
rc = _cl5AppInit ();
2948
2987
if (rc != CL5_SUCCESS)
2950
2989
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
2951
"_cl5Upgrade4_4: failed to open the db env\n");
2990
"_cl5UpgradeMinor: failed to open the db env\n");
2954
2993
s_cl5Desc.dbOpenMode = backup;
3028
3067
if (dbmajor < DB_VERSION_MAJOR)
3031
rc = _cl5Upgrade3_4(dbVersion, clVersion);
3070
rc = _cl5UpgradeMajor(dbVersion, clVersion);
3032
3071
if (rc != CL5_SUCCESS)
3034
3073
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
3040
3079
else if (dbminor < DB_VERSION_MINOR)
3042
3081
/* minor upgrade */
3043
rc = _cl5Upgrade4_4(dbVersion, clVersion);
3082
rc = _cl5UpgradeMinor(dbVersion, clVersion);
3044
3083
if (rc != CL5_SUCCESS)
3046
3085
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
3507
3546
* This change can be trimmed if it exceeds purge
3508
3547
* parameters and has been seen by all consumers.
3550
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, "_cl5TrimFile: "
3551
"Operation missing csn, moving on to next entry.\n");
3552
cl5_operation_parameters_done (&op);
3553
finished =_cl5GetNextEntry (&entry, it);
3510
3556
csn_rid = csn_get_replicaid (op.csn);
3511
3557
if ( (*numToTrim > 0 || _cl5CanTrim (entry.time, numToTrim)) &&
3512
3558
ruv_covers_csn_strict (ruv, op.csn) )
3836
3882
rc = _cl5GetFirstEntry (obj, &entry, &iterator, NULL);
3837
3883
while (rc == CL5_SUCCESS)
3839
rid = csn_get_replicaid (op.csn);
3886
rid = csn_get_replicaid (op.csn);
3888
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl, "_cl5ConstructRUV: "
3889
"Operation missing csn, moving on to next entry.\n");
3890
cl5_operation_parameters_done (&op);
3891
rc = _cl5GetNextEntry (&entry, iterator);
3840
3894
if(is_cleaned_rid(rid)){
3841
3895
/* skip this entry as the rid is invalid */
3842
3896
slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5ConstructRUV: "
5502
5556
/* Helper functions that work with individual changelog files */
5504
5558
/* file name format : <replica name>_<replica generation>db{2,3,...} */
5505
static PRBool _cl5FileName2Replica (const char *file_name, Object **replica)
5560
_cl5FileName2Replica (const char *file_name, Object **replica)
5508
5563
char *repl_name, *file_gen, *repl_gen;
5511
PR_ASSERT (file_name && replica);
5566
PR_ASSERT (file_name && replica);
5513
5568
*replica = NULL;
5515
5570
/* this is database file */
5516
5571
if (_cl5FileEndsWith (file_name, DB_EXTENSION) ||
5572
_cl5FileEndsWith (file_name, DB_EXTENSION_DB4) ||
5517
5573
_cl5FileEndsWith (file_name, DB_EXTENSION_DB3) )
5519
repl_name = slapi_ch_strdup (file_name);
5575
repl_name = slapi_ch_strdup (file_name);
5520
5576
file_gen = strstr(repl_name, FILE_SEP);
5523
int extlen = strlen(DB_EXTENSION);
5579
int extlen = strlen(DB_EXTENSION);
5524
5580
*file_gen = '\0';
5525
5581
file_gen += strlen (FILE_SEP);
5526
5582
len = strlen (file_gen);
5527
5583
if (len <= extlen + 1)
5529
slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5FileName2Replica "
5530
"invalid file name (%s)\n", file_name);
5585
slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl,
5586
"_cl5FileName2Replica "
5587
"invalid file name (%s)\n", file_name);
5542
5599
PR_ASSERT (repl_gen);
5543
5600
if (strcmp (file_gen, repl_gen) != 0)
5545
slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5FileName2Replica "
5546
"replica generation mismatch for replica at (%s), "
5602
slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl,
5603
"_cl5FileName2Replica "
5604
"replica generation mismatch for replica at (%s), "
5547
5605
"file generation %s, new replica generation %s\n",
5548
5606
slapi_sdn_get_dn (replica_get_root (r)), file_gen, repl_gen);
5560
5618
slapi_log_error(SLAPI_LOG_REPL, repl_plugin_name_cl, "_cl5FileName2Replica "
5561
"malformed file name - %s\n", file_name);
5619
"malformed file name - %s\n", file_name);
5564
5622
return PR_TRUE;
5767
5825
return CL5_UNKNOWN_ERROR;
5770
(*dbFile) = (CL5DBFile *)slapi_ch_calloc (1, sizeof (CL5DBFile));
5828
(*dbFile) = (CL5DBFile *)slapi_ch_calloc (1, sizeof (CL5DBFile));
5771
5829
if (*dbFile == NULL)
5773
5831
slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name_cl,
5775
5833
return CL5_MEMORY_ERROR;
5778
name = _cl5MakeFileName (replName, replGen);
5836
name = _cl5MakeFileName (replName, replGen);
5780
5838
/* The subname argument allows applications to have
5781
5839
* subdatabases, i.e., multiple databases inside of a single