Update FS with object sizes and print hashes

This commit is contained in:
Deon George 2014-06-20 22:51:18 +10:00
parent 7c4866dd43
commit 2a9f0c0aac
5 changed files with 146 additions and 66 deletions

View File

@ -19,26 +19,33 @@ int tsm_listfile_cb(dsmQueryType qType, DataBlk *qResp, void *userdata) {
char stor[2];
char state[3];
char ced[4];
char objInfo[DSM_MAX_OBJINFO_LENGTH];
if (userdata != NULL ) {
fprintf(stderr, "tsm_listfile_cb: Internal error: userdate != NULL");
fprintf(stderr, "tsm_listfile_cb: Internal error: userdata != NULL");
return -1;
}
if (qType == qtArchive) {
if (! headerPrinted++)
printf("%40s %2s %s %3s %19s %19s %10s %10s %s\n",
if (! headerPrinted++) {
printf("%40s %2s %s %3s %19s %19s %10s %10s %10s",
"NAME",
"ST",
"L",
"CED",
"BACKUP",
"EXPIRE",
"EST SIZE",
"SIZE",
"ID",
"DESC"
);
#ifdef USE_DIGEST
printf(" %s","OBJINFO");
#endif
printf("\n");
}
qryRespArchiveData respArchive;
memset(&respArchive,0x00,sizeof(qryRespArchiveData));
qryRespArchiveData *qr = (void *) qResp->bufferPtr;
@ -68,22 +75,29 @@ int tsm_listfile_cb(dsmQueryType qType, DataBlk *qResp, void *userdata) {
strcat(ced,respArchive.clientDeduplicated ? "D" : "-");
// The Object Status
printf("%40s|%2s|%s|%3s|%19s|%19s|%7.3f MB|%u-%u|%s\n",dsmObjnameToStr(respArchive.objName),state,stor,ced,dsmDateToStr(respArchive.insDate),respArchive.expDate.year ? dsmDateToStr(respArchive.expDate) : "",dsmSizeToNum(respArchive.sizeEstimate),respArchive.objId.hi,respArchive.objId.lo,respArchive.descr);
printf("%40s|%2s|%s|%3s|%19s|%19s|%7.3f MB|%u-%u|%s",dsmObjnameToStr(respArchive.objName),state,stor,ced,dsmDateToStr(respArchive.insDate),respArchive.expDate.year ? dsmDateToStr(respArchive.expDate) : "",dsmSizeToNum(respArchive.sizeEstimate,bTrue),respArchive.objId.hi,respArchive.objId.lo,respArchive.descr);
} else if (qType == qtBackup) {
if (! headerPrinted++)
printf("%40s %2s %s %3s %19s %19s %10s %s\n",
if (! headerPrinted++) {
printf("%40s %2s %s %3s %19s %19s %10s %10s",
"NAME",
"ST",
"L",
"CED",
"BACKUP",
"EXPIRE",
"EST SIZE",
"SIZE",
"ID"
);
#ifdef USE_DIGEST
printf(" %s","OBJINFO");
#endif
printf("\n");
}
qryRespBackupData respBackup;
memset(&respBackup,0x00,sizeof(qryRespBackupData));
qryRespBackupData *qr = (void *) qResp->bufferPtr;
respBackup = *qr;
@ -117,13 +131,25 @@ int tsm_listfile_cb(dsmQueryType qType, DataBlk *qResp, void *userdata) {
strcat(ced,respBackup.clientDeduplicated ? "D" : "-");
printf("%40s|%2s|%s|%3s|%19s|%19s|%7.3f MB|%u-%u\n",dsmObjnameToStr(respBackup.objName),state,stor,ced,dsmDateToStr(respBackup.insDate),respBackup.expDate.year ? dsmDateToStr(respBackup.expDate) : "",dsmSizeToNum(respBackup.sizeEstimate),respBackup.objId.hi,respBackup.objId.lo);
#ifdef USE_DIGEST
// Object Info
memset(&objInfo,0x00,sizeof(objInfo));
if (respBackup.objInfolen)
strncat(objInfo,respBackup.objInfo,respBackup.objInfolen);
#endif
printf("%40s|%2s|%s|%3s|%19s|%19s|%7.3f MB|%u-%8u",dsmObjnameToStr(respBackup.objName),state,stor,ced,dsmDateToStr(respBackup.insDate),respBackup.expDate.year ? dsmDateToStr(respBackup.expDate) : "",dsmSizeToNum(respBackup.sizeEstimate,bTrue),respBackup.objId.hi,respBackup.objId.lo);
} else {
fprintf(stderr,"tsm_listfile_cb: Internal error: Unknown qType %d\n",qType);
return -1;
}
#ifdef USE_DIGEST
printf("|%s",objInfo);
#endif
printf("\n");
return 1;
}

View File

@ -152,7 +152,7 @@ char *dsmObjnameToStr(dsmObjName objName) {
return s;
}
double dsmSizeToNum(dsStruct64_t dsStruct64) {
double dsmSizeToNum(dsStruct64_t dsStruct64,dsBool_t inMB) {
unsigned long long filesize=0;
filesize = 0x00;
@ -162,7 +162,7 @@ double dsmSizeToNum(dsStruct64_t dsStruct64) {
filesize |= dsStruct64.lo;
// Return number in MB
return (float)filesize/1024/1024;
return (float) inMB ? filesize/1024/1024 : filesize;
}
void debugLog(int level, _IO_FILE *output, char *message, int die) {

View File

@ -1,17 +1,5 @@
/*
* The recommended buffer size is n*TCPBUFFLEN - 4 bytes.
* To get your buffer size, do: dsmc query options|grep TCPBUF
* 32kB seems to be the new default, 31kB was the old.
*
* An additional factor is the pipe buffer size. Since we don't do threading
* (yet), we hide a little bit of latency if we don't read more than the pipe
* buffer can hold at a time. On Linux 2.6 this is 64kB.
*
* So, I would recommend a BUFLEN of 64kB if your TCPBUFLEN is above the
* 64kB + 4 bytes limit or if your TCPBUFLEN is lower, the
* n*TCPBUFFLEN - 4 bytes that gets you closest to 64kB.
*
* For a default tuned TSM client on Linux, BUFLEN should thus be 64*1024-4.
* This function controls our communication with TSM
*/
#define BUFLEN (64*1024-4)
@ -28,10 +16,6 @@
#include "../tsmpipe.h"
#ifdef USE_DIGEST
#include <openssl/evp.h>
#endif
int tsm_matchone_cb(dsmQueryType qType, DataBlk *qResp, void *userdata) {
struct matchone_cb_data *cbdata = userdata;
@ -123,9 +107,9 @@ int tsm_regfs(dsUint32_t dsmHandle, char *fsname) {
/* Send data to TSM for storage */
#ifdef USE_DIGEST
int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, char *digest) {
double tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, dsBool_t summary, char *digest) {
#else
int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype) {
double tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, dsBool_t summary) {
#endif
extern int verbose;
char *buffer;
@ -147,6 +131,7 @@ int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long l
}
#ifdef USE_DIGEST
char digest_str[EVP_MAX_MD_SIZE*2];
EVP_MD_CTX mdctx;
const EVP_MD *md=NULL;
unsigned char md_value[EVP_MAX_MD_SIZE];
@ -162,6 +147,7 @@ int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long l
return 0;
}
}
#endif
// Register our filespace
@ -296,33 +282,45 @@ int tsm_sendfile(dsUint32_t dsmHandle, char *fsname, char *filename, long long l
return 0;
}
if (verbose) {
printf("Total bytes sent: %5.3f\n",dsmSizeToNum(dsmEndSendObjExOut.totalBytesSent));
if (dsmEndSendObjExOut.objDeduplicated)
printf("Deduplicated size: %5.3f\n",dsmSizeToNum(dsmEndSendObjExOut.totalDedupSize));
if (dsmEndSendObjExOut.objCompressed)
printf("Compressed size: %5.3f\n",dsmSizeToNum(dsmEndSendObjExOut.totalCompressSize));
if (dsmSizeToNum(dsmEndSendObjExOut.totalLFBytesSent))
printf("LAN-free bytes sentsize: %5.3f\n",dsmSizeToNum(dsmEndSendObjExOut.totalLFBytesSent));
if (dsmEndSendObjExOut.encryptionType) {
printf("Encryption: %s\n",dsmEndSendObjExOut.encryptionType & DSM_ENCRYPT_CLIENTENCRKEY ? "CLIENTENCRKEY" : dsmEndSendObjExOut.encryptionType & DSM_ENCRYPT_USER ? "USER" : "NO");
printf("Encryption Strength: %s\n",dsmEndSendObjExOut.encryptionType & DSM_ENCRYPT_AES_128BIT ? "AES_128BIT" : dsmEndSendObjExOut.encryptionType & DSM_ENCRYPT_DES_56BIT ? "DES_56BIT" : "NONE");
}
}
#ifdef USE_DIGEST
if (digest) {
EVP_DigestFinal_ex(&mdctx, md_value, &md_len);
EVP_MD_CTX_cleanup(&mdctx);
for(i=0; i<md_len; i++) fprintf(stdout,"%02x",md_value[i]);
fprintf(stdout," %s (%s)\n",dsmObjnameToStr(objName),digest);
for(i=0; i<md_len; i++)
sprintf(digest_str+(i*2), "%02x", md_value[i]);
digest_str[strlen(digest_str)+1] = '\0';
fprintf(stdout,"%s %s (%s)",digest_str,dsmObjnameToStr(objName),digest);
}
objAttr.objInfo=digest_str;
objAttr.objInfoLength = md_len*2;
#endif
return 1;
// Update our object with its correct size
objAttr.sizeEstimate.hi = dsmEndSendObjExOut.totalBytesSent.hi;
objAttr.sizeEstimate.lo = dsmEndSendObjExOut.totalBytesSent.lo;
rc = dsmUpdateObj(dsmHandle,sendtype,archDataP,&objName,&objAttr,DSM_BACKUPD_OBJINFO | DSM_ARCHUPD_OBJINFO);
if (summary) {
if (! digest)
fprintf(stdout,"%s",dsmObjnameToStr(objName));
fprintf(stdout," [%5.3f/%5.3f/%5.3f/%5.3f] (%s/%s)",
dsmSizeToNum(dsmEndSendObjExOut.totalBytesSent,bTrue),
dsmEndSendObjExOut.objDeduplicated ? dsmSizeToNum(dsmEndSendObjExOut.totalDedupSize,bTrue) : 0.0,
dsmEndSendObjExOut.objCompressed ? dsmSizeToNum(dsmEndSendObjExOut.totalCompressSize,bTrue) : 0.0,
dsmSizeToNum(dsmEndSendObjExOut.totalLFBytesSent,bTrue) ? dsmSizeToNum(dsmEndSendObjExOut.totalLFBytesSent,bTrue) : 0.0,
dsmEndSendObjExOut.encryptionType ? ( DSM_ENCRYPT_CLIENTENCRKEY ? "CLIENTENCRKEY" : DSM_ENCRYPT_USER ? "USER" : "NO" ) : "-",
dsmEndSendObjExOut.encryptionType ? ( DSM_ENCRYPT_AES_128BIT ? "AES_128BIT" : DSM_ENCRYPT_DES_56BIT ? "DES_56BIT" : "NONE" ) : "-");
}
if (digest || summary)
fprintf(stdout,"\n");
return dsmSizeToNum(dsmEndSendObjExOut.totalBytesSent,bFalse);
}
/* Get data from TSM for restore */
@ -499,3 +497,23 @@ int tsm_deletefile(dsUint32_t dsmHandle, dsmQueryType qType, qryArchiveData qaDa
return 1;
}
/* Update the FS capacity and utilsation */
int tsm_updatefs(dsUint32_t dsmHandle, char *fsname, long long size ) {
dsInt16_t rc=0;
dsmFSUpd fsUpdateP;
memset(&fsUpdateP,0x00,sizeof(dsmFSUpd));
fsUpdateP.occupancy.hi = size >> 32;
fsUpdateP.occupancy.lo = size & ~0U;
fsUpdateP.capacity = fsUpdateP.occupancy;
rc = dsmUpdateFS(dsmHandle,fsname,&fsUpdateP,DSM_FSUPD_OCCUPANCY | DSM_FSUPD_CAPACITY);
if (rc != DSM_RC_OK) {
tsm_printerr(dsmHandle,rc,"tsm_updatefs: dsmUpdateFS failed");
return 0;
}
return 1;
}

View File

@ -17,8 +17,12 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
Original Idea Copyright (c) 2006,2007 HPC2N, Umeå University, Sweden
Modifications Copyright (c) 2012-2013 by Deon George
Original idea International Business Machines redook "Using ADSM to Back Up
Databases", published 1998.
Some adaption from HPC2N, Umeå University, Sweden, 2006/7
This version Copyright (c) 2012-2014 by Deon George
*/
/* Enable Large File Support stuff */
@ -65,21 +69,24 @@ void usage(void) {
fprintf(stderr,
"tsmpipe %s, usage:\n"
"\n"
"tsmpipe [-i]|[[-A|-B] [-c|-x|-d|-t] -s fsname -f filepath [-l len] ...]\n"
"tsmpipe [-i]|[[-A|-B|-U] [-c|-x|-d|-t] -s fsname -f filepath [-l len] ...]\n"
" -i Show session information:\n"
" -A and -B are mutually exclusive:\n"
" -A, -B and -U are mutually exclusive:\n"
" -A Use Archive objects\n"
" -B Use Backup objects\n"
" -U Update FS with utilisation info\n"
" -c, -x, -d and -t are mutually exclusive:\n"
" -c Create: Read from stdin and store in TSM\n"
" -x eXtract: Recall from TSM and write to stdout\n"
" -d Delete: Delete object from TSM\n"
" -t lisT: Print filelist to stdout\n"
" -s and -f are required arguments for (-A/ -B operations):\n"
" -s fsname Name of filesystem in TSM\n"
" -s fsname Name of filespace in TSM\n"
" -f filepath Path to file within filesystem in TSM\n"
" -l length Length of object to store. If guesstimating, too large\n"
" is better than too small\n"
" -L length Length of object to store. But also update the FS with\n"
" the size of the object sent\n"
" -D desc Description of archive object\n"
" -P pitdate PITDate (mmddYYYY[:HHMMSS]) (BACKUP Objects)\n"
" -n insdate Insert Date lower bound (mmddYYYY[:HHMMSS]) (ARCHIVE Objects)\n"
@ -91,6 +98,7 @@ void usage(void) {
" -m digest Calculate digest for data being stored in TSM, eg: md5, sha1...\n"
#endif
" -v Verbose. More -v's gives more verbosity\n"
" -V Verbose information on TSM transfer\n"
,_TSMPIPE_VERSION);
exit(1);
@ -108,6 +116,7 @@ int main(int argc, char *argv[]) {
extern char *optarg;
int action=0;
char *space=NULL, *filename=NULL, *lenstr=NULL, *desc=NULL, *pitdate=NULL, *options=NULL, *expdate_low=NULL, *expdate_high=NULL, *insdate_low=NULL, *insdate_high=NULL;
dsBool_t summary=bFalse;
off_t length;
dsUint32_t dsmHandle;
dsmQueryType qType=0xff; // We set this to an unlikely value, it should be set correctly via our getopt
@ -115,11 +124,12 @@ int main(int argc, char *argv[]) {
qryBackupData qbData;
dsmObjName objName;
int rc=0;
double stored=0;
#ifdef USE_DIGEST
char *digest=NULL;
#endif
while ((c = getopt(argc, argv, "hiABcxdtve:E:f:l:m:n:N:s:D:O:P:")) != -1) {
while ((c = getopt(argc, argv, "hiABcxdtUvVe:E:f:l:L:m:n:N:s:D:O:P:")) != -1) {
switch(c) {
case 'h': usage(); break;
case 'A': qType = qtArchive; break;
@ -130,11 +140,14 @@ int main(int argc, char *argv[]) {
case 'x': if (action != 0) usage_action(); action = ACTION_EXTRACT; break;
case 'd': if (action != 0) usage_action(); action = ACTION_DELETE; break;
case 't': if (action != 0) usage_action(); action = ACTION_LIST; break;
case 'U': if (action != 0) usage_action(); action = ACTION_UPDATE; break;
case 'v': verbose++; break;
case 'V': summary = bTrue; break;
case 's': space = optarg; break;
case 'f': filename = optarg; break;
case 'l': lenstr = optarg; break;
case 'L': lenstr = optarg; if (action != ACTION_CREATE) usage_action(); action = ACTION_CREATE_UPDATE; break;
case 'D': desc = optarg; break;
#ifdef USE_DIGEST
@ -179,18 +192,18 @@ int main(int argc, char *argv[]) {
exit(1);
}
if (! filename) {
if (! filename && action != ACTION_UPDATE) {
fprintf(stderr, "tsmpipe: ERROR: Must give -f filename\n");
exit(1);
}
if (action == ACTION_CREATE && ! lenstr) {
fprintf(stderr, "tsmpipe: ERROR: Must give -l length with -c\n");
if ((action == ACTION_CREATE || action == ACTION_CREATE_UPDATE || action == ACTION_UPDATE) && ! lenstr) {
fprintf(stderr, "tsmpipe: ERROR: Must give -l|-L length with -c | -u\n");
exit(1);
}
if (action != ACTION_CREATE && lenstr) {
fprintf(stderr, "tsmpipe: ERROR: -l length useless without -c\n");
if ((action != ACTION_CREATE && action != ACTION_CREATE_UPDATE && action != ACTION_UPDATE) && lenstr) {
fprintf(stderr, "tsmpipe: ERROR: -l|-L length useless without -c | -u\n");
exit(1);
}
@ -227,6 +240,7 @@ int main(int argc, char *argv[]) {
// If we are backing up or archiving
case ACTION_CREATE:
case ACTION_CREATE_UPDATE:
debugLog(2,stderr,"tsmpipe: CREATE Operation",0);
length = atof(lenstr);
@ -234,10 +248,17 @@ int main(int argc, char *argv[]) {
debugLog(0,stderr,"tsmpipe: ERROR: Provide positive length, overestimate if guessing",1);
#ifdef USE_DIGEST
rc = tsm_sendfile(dsmHandle,space,filename,length,desc,((qType == qtArchive) ? stArchiveMountWait : stBackupMountWait),digest);
stored = tsm_sendfile(dsmHandle,space,filename,length,desc,((qType == qtArchive) ? stArchiveMountWait : stBackupMountWait),summary,digest);
#else
rc = tsm_sendfile(dsmHandle,space,filename,length,desc,((qType == qtArchive) ? stArchiveMountWait : stBackupMountWait));
stored = tsm_sendfile(dsmHandle,space,filename,length,desc,((qType == qtArchive) ? stArchiveMountWait : stBackupMountWait),summary);
#endif
rc = stored ? 1 : 0;
if (rc && action == ACTION_CREATE_UPDATE && qType==qtBackup) {
debugLog(2,stderr,"tsmpipe: UPDATE FS Operation",0);
rc = tsm_updatefs(dsmHandle,space,stored);
}
break;
case ACTION_DELETE:
@ -327,6 +348,14 @@ int main(int argc, char *argv[]) {
}
break;
case ACTION_UPDATE:
length = atof(lenstr);
if (length <= 0)
debugLog(0,stderr,"tsmpipe: ERROR: Provide positive length",1);
rc = tsm_updatefs(dsmHandle,space,length);
break;
default:
fprintf(stderr,"tsmpipe: UNKNOWN Operation %d",action);
exit(2);

View File

@ -1,14 +1,20 @@
#define _TSMPIPE_VERSION "1.5.5"
#define _TSMPIPE_VERSION "1.6.0"
#define ACTION_INFO 1
#define ACTION_CREATE 2
#define ACTION_EXTRACT 3
#define ACTION_DELETE 4
#define ACTION_LIST 5
#define ACTION_UPDATE 6
#define ACTION_CREATE_UPDATE 7
// If you want to use MD5/SHA1 calculations as the data goes in here, ensure this is defined.
#define USE_DIGEST
#ifdef USE_DIGEST
#include <openssl/evp.h>
#endif
typedef int (*tsm_query_callback)(dsmQueryType, DataBlk *, void *);
struct matchone_cb_data { int numfound; dsStruct64_t objId; dsUint32_t copyGroup; };
@ -26,13 +32,14 @@ extern int tsm_deletefile (dsUint32_t dsmHandle, dsmQueryType qType, qryArchiveD
extern int tsm_listfile (dsUint32_t dsmHandle, dsmQueryType qType, qryArchiveData qaData, qryBackupData qbData);
extern int tsm_restorefile(dsUint32_t dsmHandle, dsmQueryType qType, qryArchiveData qaData, qryBackupData qbData);
#ifdef USE_DIGEST
extern int tsm_sendfile (dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, char *digest);
extern double tsm_sendfile (dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, dsBool_t summary, char *digest);
#else
extern int tsm_sendfile (dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype);
extern double tsm_sendfile (dsUint32_t dsmHandle, char *fsname, char *filename, long long length, char *description, dsmSendType sendtype, dsBoot_t summary);
#endif
extern int tsm_sessioninfo(dsUint32_t dsmHandle);
extern int tsm_updatefs(dsUint32_t dsmHandle, char *fsname, long long size);
extern char *dsmDateToStr(dsmDate date);
extern char *dsmObjnameToStr(dsmObjName objName);
extern double dsmSizeToNum(dsStruct64_t dsStruct64);
extern double dsmSizeToNum(dsStruct64_t dsStruct64,dsBool_t inMB);