diff --git a/src/Makefile b/src/Makefile index 86607c5..fdf52c7 100644 --- a/src/Makefile +++ b/src/Makefile @@ -36,7 +36,7 @@ SUNCC = CC SUNcc = cc S86OPT = -D_REENTRANT -DOO_STD $(NLGR) -DSUN -DSUNX86 -Wno-deprecated -S86OPT = $(SUNOPT) -DSUNX86 +S86OPT = $(SUNOPT) -DSUNX86 S86CC = CC S86cc = gcc @@ -77,6 +77,8 @@ SOURCE = \ bbcp_Args.C \ bbcp_BuffPool.C \ bbcp_C32.C \ + bbcp_C32C.C \ + bbcp_C32_Zip.C \ bbcp_ChkSum.C \ bbcp_Config.C \ bbcp_Emsg.C \ @@ -116,6 +118,8 @@ OBJECT = \ $(OBJDIR)/bbcp_Args.o \ $(OBJDIR)/bbcp_BuffPool.o \ $(OBJDIR)/bbcp_C32.o \ + $(OBJDIR)/bbcp_C32C.o \ + $(OBJDIR)/bbcp_C32_Zip.o \ $(OBJDIR)/bbcp_ChkSum.o \ $(OBJDIR)/bbcp_Config.o \ $(OBJDIR)/bbcp_Emsg.o \ @@ -154,7 +158,7 @@ TARGET = $(BINDIR)/bbcp all: - @make `uname` OSVER=`../MakeSname` RELEASE=`../MakeSname -r` + @+make `uname` OSVER=`../MakeSname` RELEASE=`../MakeSname -r` @echo Make done. doitall: $(TARGET) @@ -191,12 +195,12 @@ FreeBSD: LIBS="$(BSDLIBS)" Linux: - @make makeLinux`/bin/uname -i` \ + @+make makeLinux`/bin/uname -i` \ MD5HEADER=bbcp_MD5_openssl.h \ OBJMD5="$(OBJDIR)/bbcp_MD5_openssl.o" makeLinuxi386: - @make $(MKPARMS) \ + @+make $(MKPARMS) \ CC=$(LNXCC) \ BB=$(LNXcc) \ CFLAGS="$(ENVCFLAGS) $(LNXOPT)" \ @@ -204,8 +208,8 @@ makeLinuxi386: INCLUDE="$(ENVINCLUDE)" \ LIBS="$(LNXLIBS32)" -makeLinuxx86_64: - @make $(MKPARMS) RR=$(RELNUM) \ +makeLinuxx86_64 makeLinuxaarch64: + @+make $(MKPARMS) RR=$(RELNUM) \ CC=$(LNXCC) \ BB=$(LNXcc) \ CFLAGS="$(ENVCFLAGS) $(LNXOPT)" \ @@ -214,7 +218,7 @@ makeLinuxx86_64: LIBS="$(LNXLIBS64)" Darwin: - @MACOSX_DEPLOYMENT_TARGET=10.5;\ + @MACOSX_DEPLOYMENT_TARGET=10.11;\ make $(MKPARMS) \ CC=$(MACCC) \ BB=$(MACcc) \ @@ -282,8 +286,16 @@ $(OBJDIR)/bbcp_C32.o: bbcp_C32.C bbcp_C32.h bbcp_ChkSum.h bbcp_Endian.h @echo Compiling bbcp_C32.C @$(CC) -c $(CFLAGS) $(INCLUDE) $(*F).C -o $(OBJDIR)/$(*F).o +$(OBJDIR)/bbcp_C32C.o: bbcp_C32C.C bbcp_C32C.h bbcp_ChkSum.h bbcp_Endian.h + @echo Compiling bbcp_C32C.C + @$(CC) -c $(CFLAGS) $(INCLUDE) $(*F).C -o $(OBJDIR)/$(*F).o + +$(OBJDIR)/bbcp_C32_Zip.o: bbcp_C32_Zip.C bbcp_C32_Zip.h bbcp_ChkSum.h bbcp_Endian.h + @echo Compiling bbcp_C32_Zip.C + @$(CC) -c $(CFLAGS) $(INCLUDE) $(*F).C -o $(OBJDIR)/$(*F).o + $(OBJDIR)/bbcp_ChkSum.o: bbcp_ChkSum.C bbcp_ChkSum.h bbcp_Endian.h \ - bbcp_A32.h bbcp_A32_zlib.h bbcp_C32.h bbcp_MD5.h $(MD5HEADER) + bbcp_A32.h bbcp_A32_zlib.h bbcp_C32.h bbcp_C32_Zip.h bbcp_MD5.h $(MD5HEADER) @echo Compiling bbcp_ChkSum.C @$(CC) -c $(CFLAGS) $(INCLUDE) $(*F).C -o $(OBJDIR)/$(*F).o diff --git a/src/bbcp.C b/src/bbcp.C index 7577fa1..d93d78c 100644 --- a/src/bbcp.C +++ b/src/bbcp.C @@ -2,7 +2,8 @@ /* */ /* b b c p . C */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -65,7 +66,7 @@ /* G l o b a l V a r i a b l e s */ /******************************************************************************/ -extern bbcp_Config bbcp_Config; +extern bbcp_Config bbcp_Cfg; extern bbcp_BuffPool bbcp_BuffPool; @@ -90,20 +91,20 @@ int main(int argc, char *argv[], char *envp[]) // Process configuration file // bbcp_OS.EnvP = envp; - if (bbcp_Config.ConfigInit(argc, argv)) exit(1); + if (bbcp_Cfg.ConfigInit(argc, argv)) exit(1); // Process the arguments // - bbcp_Config.Arguments(argc, argv); + bbcp_Cfg.Arguments(argc, argv); // Process final source/sink actions here // - if (bbcp_Config.Options & (bbcp_SRC | bbcp_SNK)) + if (bbcp_Cfg.Options & (bbcp_SRC | bbcp_SNK)) {int retc; bbcp_ProcMon theAgent; theAgent.Start(bbcp_OS.getGrandP()); {bbcp_Node SS_Node; - retc = (bbcp_Config.Options & bbcp_SRC + retc = (bbcp_Cfg.Options & bbcp_SRC ? Protocol.Process(&SS_Node) : Protocol.Request(&SS_Node)); } @@ -113,46 +114,46 @@ int main(int argc, char *argv[], char *envp[]) // Do some debugging here // Elapsed_Timer.Start(); - if (bbcp_Debug.Trace > 2) bbcp_Config.Display(); + if (bbcp_Debug.Trace > 2) bbcp_Cfg.Display(); // Allocate the source and sink node and common protocol // Source = new bbcp_Node; Sink = new bbcp_Node; - tfs = bbcp_Config.snkSpec; + tfs = bbcp_Cfg.snkSpec; // Allocate the log file // - if (bbcp_Config.Logfn) - {bbcp_Config.MLog = new bbcp_LogFile(); - if (bbcp_Config.MLog->Open(bbcp_Config.Logfn)) exit(5); + if (bbcp_Cfg.Logfn) + {bbcp_Cfg.MLog = new bbcp_LogFile(); + if (bbcp_Cfg.MLog->Open(bbcp_Cfg.Logfn)) exit(5); } // Grab all source files for each particular user/host and copy them // retc = 0; - while(!retc && (psp = bbcp_Config.srcSpec)) + while(!retc && (psp = bbcp_Cfg.srcSpec)) {fsp = psp->next; while(fsp && Same(fsp->username, psp->username) && Same(fsp->hostname, psp->hostname)) {psp = fsp; fsp = fsp->next;} psp->next = 0; - sfs = bbcp_Config.srcSpec; - bbcp_Config.srcSpec = fsp; - if (bbcp_Config.Options & bbcp_CON2SRC) + sfs = bbcp_Cfg.srcSpec; + bbcp_Cfg.srcSpec = fsp; + if (bbcp_Cfg.Options & bbcp_CON2SRC) retc = Protocol.Schedule(Source, sfs, - (char *)bbcp_Config.SrcXeq, - (char *)bbcp_Config.SrcArg, + (char *)bbcp_Cfg.SrcXeq, + (char *)bbcp_Cfg.SrcArg, Sink, tfs, - (char *)bbcp_Config.SnkXeq, - (char *)bbcp_Config.SnkArg, Sink); + (char *)bbcp_Cfg.SnkXeq, + (char *)bbcp_Cfg.SnkArg, Sink); else retc = Protocol.Schedule(Sink, tfs, - (char *)bbcp_Config.SnkXeq, - (char *)bbcp_Config.SnkArg, + (char *)bbcp_Cfg.SnkXeq, + (char *)bbcp_Cfg.SnkArg, Source, sfs, - (char *)bbcp_Config.SrcXeq, - (char *)bbcp_Config.SrcArg, Sink); + (char *)bbcp_Cfg.SrcXeq, + (char *)bbcp_Cfg.SrcArg, Sink); TotFiles += Sink->TotFiles; TotBytes += Sink->TotBytes; } @@ -165,7 +166,7 @@ int main(int argc, char *argv[], char *envp[]) // Report final statistics if wanted // DEBUG("Ending; rc=" < #include #include "bbcp_Args.h" -#include "bbcp_Config.h" #include "bbcp_Headers.h" /******************************************************************************/ /* G l o b a l O b j e c t s */ /******************************************************************************/ -extern bbcp_Config bbcp_Config; - /******************************************************************************/ /* L o c a l C l a s s e s */ /******************************************************************************/ @@ -130,10 +128,17 @@ char *bbcp_Args::getarg(int newln) /******************************************************************************/ /* g e t o p t */ /******************************************************************************/ +extern const char *bbcp_HostName; char bbcp_Args::getopt() { - char optval, optbuff[2] = {0,0}, *optspec, *arglist, *theOpt = optbuff; + char optval, *optspec, *arglist; + +// Do some required initialization +// + optbuff[0] = 0; + optbuff[1] = 0; + theOpt = optbuff; // Check if we really have any more options // @@ -145,6 +150,7 @@ char bbcp_Args::getopt() else if (inStream) {if (MOA) arglist = arg_stream.GetLine(); else arglist = (char *)""; + if (arglist && (curopt = arg_stream.GetToken(&arglist))) {if (*curopt != '-') {arg_stream.RetToken(); curopt = 0;} else curopt++; @@ -164,14 +170,13 @@ char bbcp_Args::getopt() // Check for extended options or single letter option // - if (optp && strlen(curopt) > 2 && *curopt == '-' - && (optspec = *optp%(curopt+1))) - {theOpt = curopt; curopt = 0; - if (!(optspec = index(vopts, *optspec))) + if (optp && strlen(curopt) > 2 && *curopt == '-') + {if (!(optspec = *optp%(curopt+1)) + || !(optspec = index(vopts, *optspec))) {cerr < 0) - C32Result = (C32Result<<8) - ^ crctable[(unsigned char)((C32Result>>24)^*p++)]; + C32Result = do_crc(C32Result, p, reclen); } + +uint bbcp_C32::do_crc(uint crc, const char *p, int reclen) +{ + while(reclen-- > 0) { + crc= (crc<<8) ^ crctable[(unsigned char)((crc>>24)^*p++)]; + } + return crc; +} \ No newline at end of file diff --git a/src/bbcp_C32.h b/src/bbcp_C32.h index 4fc40d8..f47a60b 100644 --- a/src/bbcp_C32.h +++ b/src/bbcp_C32.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ C 3 2 . h */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -41,28 +42,48 @@ class bbcp_C32 : public bbcp_ChkSum { public: -void Init() {C32Result = CRC32_XINIT; TotLen = 0;} +void Init() {C32Result = CRC32_XINIT; TotLen = 0; has_final = false; } void Update(const char *Buff, int BLen); int csSize() {return sizeof(C32Result);} -char *Final(char **Text=0) + +char *csCurr(char **Text=0) { + if( has_final ) { + return (char *)&C32Result; + } + tmpcrc = finish(Text); + return (char *)&tmpcrc; + +} + +char *Final(char **Text=0) { + has_final = true; + C32Result = finish(Text); + return (char *)&C32Result; +} + +private: +uint finish(char **Text) {char buff[sizeof(long long)]; long long tLcs = TotLen; int i = 0; + has_final = true; + uint crc = C32Result; if (tLcs) {while(tLcs) {buff[i++] = tLcs & 0xff ; tLcs >>= 8;} - Update(buff, i); + crc = do_crc(crc, buff, i); } - TheResult = C32Result ^ CRC32_XOROT; + crc = crc ^ CRC32_XOROT; #ifndef BBCP_BIG_ENDIAN - TheResult = htonl(TheResult); + crc = htonl(crc); #endif - if (Text) *Text = x2a((char *)&TheResult); - return (char *)&TheResult; + if (Text) *Text = x2a((char *)&crc); + return crc; } +public: const char *Type() {return "c32";} bbcp_C32() {Init();} @@ -73,7 +94,11 @@ static const uint CRC32_XINIT = 0; static const uint CRC32_XOROT = 0xffffffff; static uint crctable[256]; uint C32Result; - uint TheResult; long long TotLen; + uint32_t tmpcrc; + bool has_final; + + uint do_crc(uint crc, const char *p, int reclen); + }; #endif diff --git a/src/bbcp_C32C.C b/src/bbcp_C32C.C new file mode 100644 index 0000000..d9457aa --- /dev/null +++ b/src/bbcp_C32C.C @@ -0,0 +1,620 @@ +/******************************************************************************/ +/* */ +/* b b c p _ C R C 3 2 c . C */ +/* */ +/* ICRAR - International Centre for Radio Astronomy Research */ +/* (c) UWA - The University of Western Australia, 2016 */ +/* Copyright by UWA (in the framework of the ICRAR) */ +/* All rights reserved */ +/* */ +/* Produced by Rodrigo Tobar based on bbcp_C32c.C */ +/* */ +/* bbcp is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* bbcp is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with bbcp in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "bbcp_C32C.h" + +// All tables and the _crc32_sw method are adapted +// from Intel's Slicing By 8 Sourceforge Project +// http://sourceforge.net/projects/slicing-by-8/ +/*++ + * + * Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved + * + * This software program is licensed subject to the BSD License, + * available at http://www.opensource.org/licenses/bsd-license.html + * + --*/ + + +const uint32_t crc_tableil8_o32[256] = +{ + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, + 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, + 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, + 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, + 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, + 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, + 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, + 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, + 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, + 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, + 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, + 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, + 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, + 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, + 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, + 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, + 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, + 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, + 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, + 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, + 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, + 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, + 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, + 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, + 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, + 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, + 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, + 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, + 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 +}; + +/* + * end of the CRC lookup table crc_tableil8_o32 + */ + + + +/* + * The following CRC lookup table was generated automagically + * using the following model parameters: + * + * Generator Polynomial = ................. 0x1EDC6F41 + * Generator Polynomial Length = .......... 32 bits + * Reflected Bits = ....................... TRUE + * Table Generation Offset = .............. 32 bits + * Number of Slices = ..................... 8 slices + * Slice Lengths = ........................ 8 8 8 8 8 8 8 8 + * Directory Name = ....................... .\ + * File Name = ............................ 8x256_tables.c + */ + +const uint32_t crc_tableil8_o40[256] = +{ + 0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899, 0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945, + 0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21, 0xD39EA264, 0xC03C3A13, 0xF4DB928A, 0xE7790AFD, + 0x3FC5F181, 0x2C6769F6, 0x1880C16F, 0x0B225918, 0x714F905D, 0x62ED082A, 0x560AA0B3, 0x45A838C4, + 0xA2D13239, 0xB173AA4E, 0x859402D7, 0x96369AA0, 0xEC5B53E5, 0xFFF9CB92, 0xCB1E630B, 0xD8BCFB7C, + 0x7F8BE302, 0x6C297B75, 0x58CED3EC, 0x4B6C4B9B, 0x310182DE, 0x22A31AA9, 0x1644B230, 0x05E62A47, + 0xE29F20BA, 0xF13DB8CD, 0xC5DA1054, 0xD6788823, 0xAC154166, 0xBFB7D911, 0x8B507188, 0x98F2E9FF, + 0x404E1283, 0x53EC8AF4, 0x670B226D, 0x74A9BA1A, 0x0EC4735F, 0x1D66EB28, 0x298143B1, 0x3A23DBC6, + 0xDD5AD13B, 0xCEF8494C, 0xFA1FE1D5, 0xE9BD79A2, 0x93D0B0E7, 0x80722890, 0xB4958009, 0xA737187E, + 0xFF17C604, 0xECB55E73, 0xD852F6EA, 0xCBF06E9D, 0xB19DA7D8, 0xA23F3FAF, 0x96D89736, 0x857A0F41, + 0x620305BC, 0x71A19DCB, 0x45463552, 0x56E4AD25, 0x2C896460, 0x3F2BFC17, 0x0BCC548E, 0x186ECCF9, + 0xC0D23785, 0xD370AFF2, 0xE797076B, 0xF4359F1C, 0x8E585659, 0x9DFACE2E, 0xA91D66B7, 0xBABFFEC0, + 0x5DC6F43D, 0x4E646C4A, 0x7A83C4D3, 0x69215CA4, 0x134C95E1, 0x00EE0D96, 0x3409A50F, 0x27AB3D78, + 0x809C2506, 0x933EBD71, 0xA7D915E8, 0xB47B8D9F, 0xCE1644DA, 0xDDB4DCAD, 0xE9537434, 0xFAF1EC43, + 0x1D88E6BE, 0x0E2A7EC9, 0x3ACDD650, 0x296F4E27, 0x53028762, 0x40A01F15, 0x7447B78C, 0x67E52FFB, + 0xBF59D487, 0xACFB4CF0, 0x981CE469, 0x8BBE7C1E, 0xF1D3B55B, 0xE2712D2C, 0xD69685B5, 0xC5341DC2, + 0x224D173F, 0x31EF8F48, 0x050827D1, 0x16AABFA6, 0x6CC776E3, 0x7F65EE94, 0x4B82460D, 0x5820DE7A, + 0xFBC3FAF9, 0xE861628E, 0xDC86CA17, 0xCF245260, 0xB5499B25, 0xA6EB0352, 0x920CABCB, 0x81AE33BC, + 0x66D73941, 0x7575A136, 0x419209AF, 0x523091D8, 0x285D589D, 0x3BFFC0EA, 0x0F186873, 0x1CBAF004, + 0xC4060B78, 0xD7A4930F, 0xE3433B96, 0xF0E1A3E1, 0x8A8C6AA4, 0x992EF2D3, 0xADC95A4A, 0xBE6BC23D, + 0x5912C8C0, 0x4AB050B7, 0x7E57F82E, 0x6DF56059, 0x1798A91C, 0x043A316B, 0x30DD99F2, 0x237F0185, + 0x844819FB, 0x97EA818C, 0xA30D2915, 0xB0AFB162, 0xCAC27827, 0xD960E050, 0xED8748C9, 0xFE25D0BE, + 0x195CDA43, 0x0AFE4234, 0x3E19EAAD, 0x2DBB72DA, 0x57D6BB9F, 0x447423E8, 0x70938B71, 0x63311306, + 0xBB8DE87A, 0xA82F700D, 0x9CC8D894, 0x8F6A40E3, 0xF50789A6, 0xE6A511D1, 0xD242B948, 0xC1E0213F, + 0x26992BC2, 0x353BB3B5, 0x01DC1B2C, 0x127E835B, 0x68134A1E, 0x7BB1D269, 0x4F567AF0, 0x5CF4E287, + 0x04D43CFD, 0x1776A48A, 0x23910C13, 0x30339464, 0x4A5E5D21, 0x59FCC556, 0x6D1B6DCF, 0x7EB9F5B8, + 0x99C0FF45, 0x8A626732, 0xBE85CFAB, 0xAD2757DC, 0xD74A9E99, 0xC4E806EE, 0xF00FAE77, 0xE3AD3600, + 0x3B11CD7C, 0x28B3550B, 0x1C54FD92, 0x0FF665E5, 0x759BACA0, 0x663934D7, 0x52DE9C4E, 0x417C0439, + 0xA6050EC4, 0xB5A796B3, 0x81403E2A, 0x92E2A65D, 0xE88F6F18, 0xFB2DF76F, 0xCFCA5FF6, 0xDC68C781, + 0x7B5FDFFF, 0x68FD4788, 0x5C1AEF11, 0x4FB87766, 0x35D5BE23, 0x26772654, 0x12908ECD, 0x013216BA, + 0xE64B1C47, 0xF5E98430, 0xC10E2CA9, 0xD2ACB4DE, 0xA8C17D9B, 0xBB63E5EC, 0x8F844D75, 0x9C26D502, + 0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7, 0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B, + 0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F, 0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483 +}; + +/* + * end of the CRC lookup table crc_tableil8_o40 + */ + + + +/* + * The following CRC lookup table was generated automagically + * using the following model parameters: + * + * Generator Polynomial = ................. 0x1EDC6F41 + * Generator Polynomial Length = .......... 32 bits + * Reflected Bits = ....................... TRUE + * Table Generation Offset = .............. 32 bits + * Number of Slices = ..................... 8 slices + * Slice Lengths = ........................ 8 8 8 8 8 8 8 8 + * Directory Name = ....................... .\ + * File Name = ............................ 8x256_tables.c + */ + +const uint32_t crc_tableil8_o48[256] = +{ + 0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073, 0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469, + 0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6, 0xA68F9ADF, 0x03CE08A1, 0xE9E0C8D2, 0x4CA15AAC, + 0x70A27D8A, 0xD5E3EFF4, 0x3FCD2F87, 0x9A8CBDF9, 0xEE7CD990, 0x4B3D4BEE, 0xA1138B9D, 0x045219E3, + 0x48F3434F, 0xEDB2D131, 0x079C1142, 0xA2DD833C, 0xD62DE755, 0x736C752B, 0x9942B558, 0x3C032726, + 0xE144FB14, 0x4405696A, 0xAE2BA919, 0x0B6A3B67, 0x7F9A5F0E, 0xDADBCD70, 0x30F50D03, 0x95B49F7D, + 0xD915C5D1, 0x7C5457AF, 0x967A97DC, 0x333B05A2, 0x47CB61CB, 0xE28AF3B5, 0x08A433C6, 0xADE5A1B8, + 0x91E6869E, 0x34A714E0, 0xDE89D493, 0x7BC846ED, 0x0F382284, 0xAA79B0FA, 0x40577089, 0xE516E2F7, + 0xA9B7B85B, 0x0CF62A25, 0xE6D8EA56, 0x43997828, 0x37691C41, 0x92288E3F, 0x78064E4C, 0xDD47DC32, + 0xC76580D9, 0x622412A7, 0x880AD2D4, 0x2D4B40AA, 0x59BB24C3, 0xFCFAB6BD, 0x16D476CE, 0xB395E4B0, + 0xFF34BE1C, 0x5A752C62, 0xB05BEC11, 0x151A7E6F, 0x61EA1A06, 0xC4AB8878, 0x2E85480B, 0x8BC4DA75, + 0xB7C7FD53, 0x12866F2D, 0xF8A8AF5E, 0x5DE93D20, 0x29195949, 0x8C58CB37, 0x66760B44, 0xC337993A, + 0x8F96C396, 0x2AD751E8, 0xC0F9919B, 0x65B803E5, 0x1148678C, 0xB409F5F2, 0x5E273581, 0xFB66A7FF, + 0x26217BCD, 0x8360E9B3, 0x694E29C0, 0xCC0FBBBE, 0xB8FFDFD7, 0x1DBE4DA9, 0xF7908DDA, 0x52D11FA4, + 0x1E704508, 0xBB31D776, 0x511F1705, 0xF45E857B, 0x80AEE112, 0x25EF736C, 0xCFC1B31F, 0x6A802161, + 0x56830647, 0xF3C29439, 0x19EC544A, 0xBCADC634, 0xC85DA25D, 0x6D1C3023, 0x8732F050, 0x2273622E, + 0x6ED23882, 0xCB93AAFC, 0x21BD6A8F, 0x84FCF8F1, 0xF00C9C98, 0x554D0EE6, 0xBF63CE95, 0x1A225CEB, + 0x8B277743, 0x2E66E53D, 0xC448254E, 0x6109B730, 0x15F9D359, 0xB0B84127, 0x5A968154, 0xFFD7132A, + 0xB3764986, 0x1637DBF8, 0xFC191B8B, 0x595889F5, 0x2DA8ED9C, 0x88E97FE2, 0x62C7BF91, 0xC7862DEF, + 0xFB850AC9, 0x5EC498B7, 0xB4EA58C4, 0x11ABCABA, 0x655BAED3, 0xC01A3CAD, 0x2A34FCDE, 0x8F756EA0, + 0xC3D4340C, 0x6695A672, 0x8CBB6601, 0x29FAF47F, 0x5D0A9016, 0xF84B0268, 0x1265C21B, 0xB7245065, + 0x6A638C57, 0xCF221E29, 0x250CDE5A, 0x804D4C24, 0xF4BD284D, 0x51FCBA33, 0xBBD27A40, 0x1E93E83E, + 0x5232B292, 0xF77320EC, 0x1D5DE09F, 0xB81C72E1, 0xCCEC1688, 0x69AD84F6, 0x83834485, 0x26C2D6FB, + 0x1AC1F1DD, 0xBF8063A3, 0x55AEA3D0, 0xF0EF31AE, 0x841F55C7, 0x215EC7B9, 0xCB7007CA, 0x6E3195B4, + 0x2290CF18, 0x87D15D66, 0x6DFF9D15, 0xC8BE0F6B, 0xBC4E6B02, 0x190FF97C, 0xF321390F, 0x5660AB71, + 0x4C42F79A, 0xE90365E4, 0x032DA597, 0xA66C37E9, 0xD29C5380, 0x77DDC1FE, 0x9DF3018D, 0x38B293F3, + 0x7413C95F, 0xD1525B21, 0x3B7C9B52, 0x9E3D092C, 0xEACD6D45, 0x4F8CFF3B, 0xA5A23F48, 0x00E3AD36, + 0x3CE08A10, 0x99A1186E, 0x738FD81D, 0xD6CE4A63, 0xA23E2E0A, 0x077FBC74, 0xED517C07, 0x4810EE79, + 0x04B1B4D5, 0xA1F026AB, 0x4BDEE6D8, 0xEE9F74A6, 0x9A6F10CF, 0x3F2E82B1, 0xD50042C2, 0x7041D0BC, + 0xAD060C8E, 0x08479EF0, 0xE2695E83, 0x4728CCFD, 0x33D8A894, 0x96993AEA, 0x7CB7FA99, 0xD9F668E7, + 0x9557324B, 0x3016A035, 0xDA386046, 0x7F79F238, 0x0B899651, 0xAEC8042F, 0x44E6C45C, 0xE1A75622, + 0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177, 0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D, + 0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2, 0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8 +}; + +/* + * end of the CRC lookup table crc_tableil8_o48 + */ + + + +/* + * The following CRC lookup table was generated automagically + * using the following model parameters: + * + * Generator Polynomial = ................. 0x1EDC6F41 + * Generator Polynomial Length = .......... 32 bits + * Reflected Bits = ....................... TRUE + * Table Generation Offset = .............. 32 bits + * Number of Slices = ..................... 8 slices + * Slice Lengths = ........................ 8 8 8 8 8 8 8 8 + * Directory Name = ....................... .\ + * File Name = ............................ 8x256_tables.c + */ + +const uint32_t crc_tableil8_o56[256] = +{ + 0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939, 0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA, + 0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF, 0x8D665215, 0x5023F8AD, 0x32017194, 0xEF44DB2C, + 0xE964B13D, 0x34211B85, 0x560392BC, 0x8B463804, 0x924680CE, 0x4F032A76, 0x2D21A34F, 0xF06409F7, + 0x1F20D2DB, 0xC2657863, 0xA047F15A, 0x7D025BE2, 0x6402E328, 0xB9474990, 0xDB65C0A9, 0x06206A11, + 0xD725148B, 0x0A60BE33, 0x6842370A, 0xB5079DB2, 0xAC072578, 0x71428FC0, 0x136006F9, 0xCE25AC41, + 0x2161776D, 0xFC24DDD5, 0x9E0654EC, 0x4343FE54, 0x5A43469E, 0x8706EC26, 0xE524651F, 0x3861CFA7, + 0x3E41A5B6, 0xE3040F0E, 0x81268637, 0x5C632C8F, 0x45639445, 0x98263EFD, 0xFA04B7C4, 0x27411D7C, + 0xC805C650, 0x15406CE8, 0x7762E5D1, 0xAA274F69, 0xB327F7A3, 0x6E625D1B, 0x0C40D422, 0xD1057E9A, + 0xABA65FE7, 0x76E3F55F, 0x14C17C66, 0xC984D6DE, 0xD0846E14, 0x0DC1C4AC, 0x6FE34D95, 0xB2A6E72D, + 0x5DE23C01, 0x80A796B9, 0xE2851F80, 0x3FC0B538, 0x26C00DF2, 0xFB85A74A, 0x99A72E73, 0x44E284CB, + 0x42C2EEDA, 0x9F874462, 0xFDA5CD5B, 0x20E067E3, 0x39E0DF29, 0xE4A57591, 0x8687FCA8, 0x5BC25610, + 0xB4868D3C, 0x69C32784, 0x0BE1AEBD, 0xD6A40405, 0xCFA4BCCF, 0x12E11677, 0x70C39F4E, 0xAD8635F6, + 0x7C834B6C, 0xA1C6E1D4, 0xC3E468ED, 0x1EA1C255, 0x07A17A9F, 0xDAE4D027, 0xB8C6591E, 0x6583F3A6, + 0x8AC7288A, 0x57828232, 0x35A00B0B, 0xE8E5A1B3, 0xF1E51979, 0x2CA0B3C1, 0x4E823AF8, 0x93C79040, + 0x95E7FA51, 0x48A250E9, 0x2A80D9D0, 0xF7C57368, 0xEEC5CBA2, 0x3380611A, 0x51A2E823, 0x8CE7429B, + 0x63A399B7, 0xBEE6330F, 0xDCC4BA36, 0x0181108E, 0x1881A844, 0xC5C402FC, 0xA7E68BC5, 0x7AA3217D, + 0x52A0C93F, 0x8FE56387, 0xEDC7EABE, 0x30824006, 0x2982F8CC, 0xF4C75274, 0x96E5DB4D, 0x4BA071F5, + 0xA4E4AAD9, 0x79A10061, 0x1B838958, 0xC6C623E0, 0xDFC69B2A, 0x02833192, 0x60A1B8AB, 0xBDE41213, + 0xBBC47802, 0x6681D2BA, 0x04A35B83, 0xD9E6F13B, 0xC0E649F1, 0x1DA3E349, 0x7F816A70, 0xA2C4C0C8, + 0x4D801BE4, 0x90C5B15C, 0xF2E73865, 0x2FA292DD, 0x36A22A17, 0xEBE780AF, 0x89C50996, 0x5480A32E, + 0x8585DDB4, 0x58C0770C, 0x3AE2FE35, 0xE7A7548D, 0xFEA7EC47, 0x23E246FF, 0x41C0CFC6, 0x9C85657E, + 0x73C1BE52, 0xAE8414EA, 0xCCA69DD3, 0x11E3376B, 0x08E38FA1, 0xD5A62519, 0xB784AC20, 0x6AC10698, + 0x6CE16C89, 0xB1A4C631, 0xD3864F08, 0x0EC3E5B0, 0x17C35D7A, 0xCA86F7C2, 0xA8A47EFB, 0x75E1D443, + 0x9AA50F6F, 0x47E0A5D7, 0x25C22CEE, 0xF8878656, 0xE1873E9C, 0x3CC29424, 0x5EE01D1D, 0x83A5B7A5, + 0xF90696D8, 0x24433C60, 0x4661B559, 0x9B241FE1, 0x8224A72B, 0x5F610D93, 0x3D4384AA, 0xE0062E12, + 0x0F42F53E, 0xD2075F86, 0xB025D6BF, 0x6D607C07, 0x7460C4CD, 0xA9256E75, 0xCB07E74C, 0x16424DF4, + 0x106227E5, 0xCD278D5D, 0xAF050464, 0x7240AEDC, 0x6B401616, 0xB605BCAE, 0xD4273597, 0x09629F2F, + 0xE6264403, 0x3B63EEBB, 0x59416782, 0x8404CD3A, 0x9D0475F0, 0x4041DF48, 0x22635671, 0xFF26FCC9, + 0x2E238253, 0xF36628EB, 0x9144A1D2, 0x4C010B6A, 0x5501B3A0, 0x88441918, 0xEA669021, 0x37233A99, + 0xD867E1B5, 0x05224B0D, 0x6700C234, 0xBA45688C, 0xA345D046, 0x7E007AFE, 0x1C22F3C7, 0xC167597F, + 0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57, 0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4, + 0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1, 0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842 +}; + +/* + * end of the CRC lookup table crc_tableil8_o56 + */ + + + +/* + * The following CRC lookup table was generated automagically + * using the following model parameters: + * + * Generator Polynomial = ................. 0x1EDC6F41 + * Generator Polynomial Length = .......... 32 bits + * Reflected Bits = ....................... TRUE + * Table Generation Offset = .............. 32 bits + * Number of Slices = ..................... 8 slices + * Slice Lengths = ........................ 8 8 8 8 8 8 8 8 + * Directory Name = ....................... .\ + * File Name = ............................ 8x256_tables.c + */ + +const uint32_t crc_tableil8_o64[256] = +{ + 0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4, 0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44, + 0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65, 0x2522B521, 0x1D33DA8D, 0x55006A79, 0x6D1105D5, + 0x8F2261D3, 0xB7330E7F, 0xFF00BE8B, 0xC711D127, 0x6F67DF63, 0x5776B0CF, 0x1F45003B, 0x27546F97, + 0x4A456A42, 0x725405EE, 0x3A67B51A, 0x0276DAB6, 0xAA00D4F2, 0x9211BB5E, 0xDA220BAA, 0xE2336406, + 0x1BA8B557, 0x23B9DAFB, 0x6B8A6A0F, 0x539B05A3, 0xFBED0BE7, 0xC3FC644B, 0x8BCFD4BF, 0xB3DEBB13, + 0xDECFBEC6, 0xE6DED16A, 0xAEED619E, 0x96FC0E32, 0x3E8A0076, 0x069B6FDA, 0x4EA8DF2E, 0x76B9B082, + 0x948AD484, 0xAC9BBB28, 0xE4A80BDC, 0xDCB96470, 0x74CF6A34, 0x4CDE0598, 0x04EDB56C, 0x3CFCDAC0, + 0x51EDDF15, 0x69FCB0B9, 0x21CF004D, 0x19DE6FE1, 0xB1A861A5, 0x89B90E09, 0xC18ABEFD, 0xF99BD151, + 0x37516AAE, 0x0F400502, 0x4773B5F6, 0x7F62DA5A, 0xD714D41E, 0xEF05BBB2, 0xA7360B46, 0x9F2764EA, + 0xF236613F, 0xCA270E93, 0x8214BE67, 0xBA05D1CB, 0x1273DF8F, 0x2A62B023, 0x625100D7, 0x5A406F7B, + 0xB8730B7D, 0x806264D1, 0xC851D425, 0xF040BB89, 0x5836B5CD, 0x6027DA61, 0x28146A95, 0x10050539, + 0x7D1400EC, 0x45056F40, 0x0D36DFB4, 0x3527B018, 0x9D51BE5C, 0xA540D1F0, 0xED736104, 0xD5620EA8, + 0x2CF9DFF9, 0x14E8B055, 0x5CDB00A1, 0x64CA6F0D, 0xCCBC6149, 0xF4AD0EE5, 0xBC9EBE11, 0x848FD1BD, + 0xE99ED468, 0xD18FBBC4, 0x99BC0B30, 0xA1AD649C, 0x09DB6AD8, 0x31CA0574, 0x79F9B580, 0x41E8DA2C, + 0xA3DBBE2A, 0x9BCAD186, 0xD3F96172, 0xEBE80EDE, 0x439E009A, 0x7B8F6F36, 0x33BCDFC2, 0x0BADB06E, + 0x66BCB5BB, 0x5EADDA17, 0x169E6AE3, 0x2E8F054F, 0x86F90B0B, 0xBEE864A7, 0xF6DBD453, 0xCECABBFF, + 0x6EA2D55C, 0x56B3BAF0, 0x1E800A04, 0x269165A8, 0x8EE76BEC, 0xB6F60440, 0xFEC5B4B4, 0xC6D4DB18, + 0xABC5DECD, 0x93D4B161, 0xDBE70195, 0xE3F66E39, 0x4B80607D, 0x73910FD1, 0x3BA2BF25, 0x03B3D089, + 0xE180B48F, 0xD991DB23, 0x91A26BD7, 0xA9B3047B, 0x01C50A3F, 0x39D46593, 0x71E7D567, 0x49F6BACB, + 0x24E7BF1E, 0x1CF6D0B2, 0x54C56046, 0x6CD40FEA, 0xC4A201AE, 0xFCB36E02, 0xB480DEF6, 0x8C91B15A, + 0x750A600B, 0x4D1B0FA7, 0x0528BF53, 0x3D39D0FF, 0x954FDEBB, 0xAD5EB117, 0xE56D01E3, 0xDD7C6E4F, + 0xB06D6B9A, 0x887C0436, 0xC04FB4C2, 0xF85EDB6E, 0x5028D52A, 0x6839BA86, 0x200A0A72, 0x181B65DE, + 0xFA2801D8, 0xC2396E74, 0x8A0ADE80, 0xB21BB12C, 0x1A6DBF68, 0x227CD0C4, 0x6A4F6030, 0x525E0F9C, + 0x3F4F0A49, 0x075E65E5, 0x4F6DD511, 0x777CBABD, 0xDF0AB4F9, 0xE71BDB55, 0xAF286BA1, 0x9739040D, + 0x59F3BFF2, 0x61E2D05E, 0x29D160AA, 0x11C00F06, 0xB9B60142, 0x81A76EEE, 0xC994DE1A, 0xF185B1B6, + 0x9C94B463, 0xA485DBCF, 0xECB66B3B, 0xD4A70497, 0x7CD10AD3, 0x44C0657F, 0x0CF3D58B, 0x34E2BA27, + 0xD6D1DE21, 0xEEC0B18D, 0xA6F30179, 0x9EE26ED5, 0x36946091, 0x0E850F3D, 0x46B6BFC9, 0x7EA7D065, + 0x13B6D5B0, 0x2BA7BA1C, 0x63940AE8, 0x5B856544, 0xF3F36B00, 0xCBE204AC, 0x83D1B458, 0xBBC0DBF4, + 0x425B0AA5, 0x7A4A6509, 0x3279D5FD, 0x0A68BA51, 0xA21EB415, 0x9A0FDBB9, 0xD23C6B4D, 0xEA2D04E1, + 0x873C0134, 0xBF2D6E98, 0xF71EDE6C, 0xCF0FB1C0, 0x6779BF84, 0x5F68D028, 0x175B60DC, 0x2F4A0F70, + 0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82, 0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532, + 0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013, 0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3 +}; + +/* + * end of the CRC lookup table crc_tableil8_o64 + */ + + + +/* + * The following CRC lookup table was generated automagically + * using the following model parameters: + * + * Generator Polynomial = ................. 0x1EDC6F41 + * Generator Polynomial Length = .......... 32 bits + * Reflected Bits = ....................... TRUE + * Table Generation Offset = .............. 32 bits + * Number of Slices = ..................... 8 slices + * Slice Lengths = ........................ 8 8 8 8 8 8 8 8 + * Directory Name = ....................... .\ + * File Name = ............................ 8x256_tables.c + */ + +const uint32_t crc_tableil8_o72[256] = +{ + 0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA, 0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD, + 0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5, 0xD2F32F68, 0x3DC34471, 0x097F8FAB, 0xE64FE4B2, + 0xC00C303E, 0x2F3C5B27, 0x1B8090FD, 0xF4B0FBE4, 0x72F90749, 0x9DC96C50, 0xA975A78A, 0x4645CC93, + 0xA00A2821, 0x4F3A4338, 0x7B8688E2, 0x94B6E3FB, 0x12FF1F56, 0xFDCF744F, 0xC973BF95, 0x2643D48C, + 0x85F4168D, 0x6AC47D94, 0x5E78B64E, 0xB148DD57, 0x370121FA, 0xD8314AE3, 0xEC8D8139, 0x03BDEA20, + 0xE5F20E92, 0x0AC2658B, 0x3E7EAE51, 0xD14EC548, 0x570739E5, 0xB83752FC, 0x8C8B9926, 0x63BBF23F, + 0x45F826B3, 0xAAC84DAA, 0x9E748670, 0x7144ED69, 0xF70D11C4, 0x183D7ADD, 0x2C81B107, 0xC3B1DA1E, + 0x25FE3EAC, 0xCACE55B5, 0xFE729E6F, 0x1142F576, 0x970B09DB, 0x783B62C2, 0x4C87A918, 0xA3B7C201, + 0x0E045BEB, 0xE13430F2, 0xD588FB28, 0x3AB89031, 0xBCF16C9C, 0x53C10785, 0x677DCC5F, 0x884DA746, + 0x6E0243F4, 0x813228ED, 0xB58EE337, 0x5ABE882E, 0xDCF77483, 0x33C71F9A, 0x077BD440, 0xE84BBF59, + 0xCE086BD5, 0x213800CC, 0x1584CB16, 0xFAB4A00F, 0x7CFD5CA2, 0x93CD37BB, 0xA771FC61, 0x48419778, + 0xAE0E73CA, 0x413E18D3, 0x7582D309, 0x9AB2B810, 0x1CFB44BD, 0xF3CB2FA4, 0xC777E47E, 0x28478F67, + 0x8BF04D66, 0x64C0267F, 0x507CEDA5, 0xBF4C86BC, 0x39057A11, 0xD6351108, 0xE289DAD2, 0x0DB9B1CB, + 0xEBF65579, 0x04C63E60, 0x307AF5BA, 0xDF4A9EA3, 0x5903620E, 0xB6330917, 0x828FC2CD, 0x6DBFA9D4, + 0x4BFC7D58, 0xA4CC1641, 0x9070DD9B, 0x7F40B682, 0xF9094A2F, 0x16392136, 0x2285EAEC, 0xCDB581F5, + 0x2BFA6547, 0xC4CA0E5E, 0xF076C584, 0x1F46AE9D, 0x990F5230, 0x763F3929, 0x4283F2F3, 0xADB399EA, + 0x1C08B7D6, 0xF338DCCF, 0xC7841715, 0x28B47C0C, 0xAEFD80A1, 0x41CDEBB8, 0x75712062, 0x9A414B7B, + 0x7C0EAFC9, 0x933EC4D0, 0xA7820F0A, 0x48B26413, 0xCEFB98BE, 0x21CBF3A7, 0x1577387D, 0xFA475364, + 0xDC0487E8, 0x3334ECF1, 0x0788272B, 0xE8B84C32, 0x6EF1B09F, 0x81C1DB86, 0xB57D105C, 0x5A4D7B45, + 0xBC029FF7, 0x5332F4EE, 0x678E3F34, 0x88BE542D, 0x0EF7A880, 0xE1C7C399, 0xD57B0843, 0x3A4B635A, + 0x99FCA15B, 0x76CCCA42, 0x42700198, 0xAD406A81, 0x2B09962C, 0xC439FD35, 0xF08536EF, 0x1FB55DF6, + 0xF9FAB944, 0x16CAD25D, 0x22761987, 0xCD46729E, 0x4B0F8E33, 0xA43FE52A, 0x90832EF0, 0x7FB345E9, + 0x59F09165, 0xB6C0FA7C, 0x827C31A6, 0x6D4C5ABF, 0xEB05A612, 0x0435CD0B, 0x308906D1, 0xDFB96DC8, + 0x39F6897A, 0xD6C6E263, 0xE27A29B9, 0x0D4A42A0, 0x8B03BE0D, 0x6433D514, 0x508F1ECE, 0xBFBF75D7, + 0x120CEC3D, 0xFD3C8724, 0xC9804CFE, 0x26B027E7, 0xA0F9DB4A, 0x4FC9B053, 0x7B757B89, 0x94451090, + 0x720AF422, 0x9D3A9F3B, 0xA98654E1, 0x46B63FF8, 0xC0FFC355, 0x2FCFA84C, 0x1B736396, 0xF443088F, + 0xD200DC03, 0x3D30B71A, 0x098C7CC0, 0xE6BC17D9, 0x60F5EB74, 0x8FC5806D, 0xBB794BB7, 0x544920AE, + 0xB206C41C, 0x5D36AF05, 0x698A64DF, 0x86BA0FC6, 0x00F3F36B, 0xEFC39872, 0xDB7F53A8, 0x344F38B1, + 0x97F8FAB0, 0x78C891A9, 0x4C745A73, 0xA344316A, 0x250DCDC7, 0xCA3DA6DE, 0xFE816D04, 0x11B1061D, + 0xF7FEE2AF, 0x18CE89B6, 0x2C72426C, 0xC3422975, 0x450BD5D8, 0xAA3BBEC1, 0x9E87751B, 0x71B71E02, + 0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154, 0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623, + 0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B, 0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C +}; + +/* + * end of the CRC lookup table crc_tableil8_o72 + */ + + + +/* + * The following CRC lookup table was generated automagically + * using the following model parameters: + * + * Generator Polynomial = ................. 0x1EDC6F41 + * Generator Polynomial Length = .......... 32 bits + * Reflected Bits = ....................... TRUE + * Table Generation Offset = .............. 32 bits + * Number of Slices = ..................... 8 slices + * Slice Lengths = ........................ 8 8 8 8 8 8 8 8 + * Directory Name = ....................... .\ + * File Name = ............................ 8x256_tables.c + */ + +const uint32_t crc_tableil8_o80[256] = +{ + 0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558, 0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089, + 0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B, 0xEBCD3882, 0x83CE144A, 0x3BCB6112, 0x53C84DDA, + 0x9C5BFAA6, 0xF458D66E, 0x4C5DA336, 0x245E8FFE, 0x39BB3F77, 0x51B813BF, 0xE9BD66E7, 0x81BE4A2F, + 0xD27607F5, 0xBA752B3D, 0x02705E65, 0x6A7372AD, 0x7796C224, 0x1F95EEEC, 0xA7909BB4, 0xCF93B77C, + 0x3D5B83BD, 0x5558AF75, 0xED5DDA2D, 0x855EF6E5, 0x98BB466C, 0xF0B86AA4, 0x48BD1FFC, 0x20BE3334, + 0x73767EEE, 0x1B755226, 0xA370277E, 0xCB730BB6, 0xD696BB3F, 0xBE9597F7, 0x0690E2AF, 0x6E93CE67, + 0xA100791B, 0xC90355D3, 0x7106208B, 0x19050C43, 0x04E0BCCA, 0x6CE39002, 0xD4E6E55A, 0xBCE5C992, + 0xEF2D8448, 0x872EA880, 0x3F2BDDD8, 0x5728F110, 0x4ACD4199, 0x22CE6D51, 0x9ACB1809, 0xF2C834C1, + 0x7AB7077A, 0x12B42BB2, 0xAAB15EEA, 0xC2B27222, 0xDF57C2AB, 0xB754EE63, 0x0F519B3B, 0x6752B7F3, + 0x349AFA29, 0x5C99D6E1, 0xE49CA3B9, 0x8C9F8F71, 0x917A3FF8, 0xF9791330, 0x417C6668, 0x297F4AA0, + 0xE6ECFDDC, 0x8EEFD114, 0x36EAA44C, 0x5EE98884, 0x430C380D, 0x2B0F14C5, 0x930A619D, 0xFB094D55, + 0xA8C1008F, 0xC0C22C47, 0x78C7591F, 0x10C475D7, 0x0D21C55E, 0x6522E996, 0xDD279CCE, 0xB524B006, + 0x47EC84C7, 0x2FEFA80F, 0x97EADD57, 0xFFE9F19F, 0xE20C4116, 0x8A0F6DDE, 0x320A1886, 0x5A09344E, + 0x09C17994, 0x61C2555C, 0xD9C72004, 0xB1C40CCC, 0xAC21BC45, 0xC422908D, 0x7C27E5D5, 0x1424C91D, + 0xDBB77E61, 0xB3B452A9, 0x0BB127F1, 0x63B20B39, 0x7E57BBB0, 0x16549778, 0xAE51E220, 0xC652CEE8, + 0x959A8332, 0xFD99AFFA, 0x459CDAA2, 0x2D9FF66A, 0x307A46E3, 0x58796A2B, 0xE07C1F73, 0x887F33BB, + 0xF56E0EF4, 0x9D6D223C, 0x25685764, 0x4D6B7BAC, 0x508ECB25, 0x388DE7ED, 0x808892B5, 0xE88BBE7D, + 0xBB43F3A7, 0xD340DF6F, 0x6B45AA37, 0x034686FF, 0x1EA33676, 0x76A01ABE, 0xCEA56FE6, 0xA6A6432E, + 0x6935F452, 0x0136D89A, 0xB933ADC2, 0xD130810A, 0xCCD53183, 0xA4D61D4B, 0x1CD36813, 0x74D044DB, + 0x27180901, 0x4F1B25C9, 0xF71E5091, 0x9F1D7C59, 0x82F8CCD0, 0xEAFBE018, 0x52FE9540, 0x3AFDB988, + 0xC8358D49, 0xA036A181, 0x1833D4D9, 0x7030F811, 0x6DD54898, 0x05D66450, 0xBDD31108, 0xD5D03DC0, + 0x8618701A, 0xEE1B5CD2, 0x561E298A, 0x3E1D0542, 0x23F8B5CB, 0x4BFB9903, 0xF3FEEC5B, 0x9BFDC093, + 0x546E77EF, 0x3C6D5B27, 0x84682E7F, 0xEC6B02B7, 0xF18EB23E, 0x998D9EF6, 0x2188EBAE, 0x498BC766, + 0x1A438ABC, 0x7240A674, 0xCA45D32C, 0xA246FFE4, 0xBFA34F6D, 0xD7A063A5, 0x6FA516FD, 0x07A63A35, + 0x8FD9098E, 0xE7DA2546, 0x5FDF501E, 0x37DC7CD6, 0x2A39CC5F, 0x423AE097, 0xFA3F95CF, 0x923CB907, + 0xC1F4F4DD, 0xA9F7D815, 0x11F2AD4D, 0x79F18185, 0x6414310C, 0x0C171DC4, 0xB412689C, 0xDC114454, + 0x1382F328, 0x7B81DFE0, 0xC384AAB8, 0xAB878670, 0xB66236F9, 0xDE611A31, 0x66646F69, 0x0E6743A1, + 0x5DAF0E7B, 0x35AC22B3, 0x8DA957EB, 0xE5AA7B23, 0xF84FCBAA, 0x904CE762, 0x2849923A, 0x404ABEF2, + 0xB2828A33, 0xDA81A6FB, 0x6284D3A3, 0x0A87FF6B, 0x17624FE2, 0x7F61632A, 0xC7641672, 0xAF673ABA, + 0xFCAF7760, 0x94AC5BA8, 0x2CA92EF0, 0x44AA0238, 0x594FB2B1, 0x314C9E79, 0x8949EB21, 0xE14AC7E9, + 0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD, 0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C, + 0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E, 0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F +}; + +/* + * end of the CRC lookup table crc_tableil8_o80 + */ + + + +/* + * The following CRC lookup table was generated automagically + * using the following model parameters: + * + * Generator Polynomial = ................. 0x1EDC6F41 + * Generator Polynomial Length = .......... 32 bits + * Reflected Bits = ....................... TRUE + * Table Generation Offset = .............. 32 bits + * Number of Slices = ..................... 8 slices + * Slice Lengths = ........................ 8 8 8 8 8 8 8 8 + * Directory Name = ....................... .\ + * File Name = ............................ 8x256_tables.c + */ + +const uint32_t crc_tableil8_o88[256] = +{ + 0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769, 0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504, + 0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3, 0x632686B7, 0x2A1AFB90, 0xF15E7CF9, 0xB86201DE, + 0x847609B4, 0xCD4A7493, 0x160EF3FA, 0x5F328EDD, 0xA56B8BD9, 0xEC57F6FE, 0x37137197, 0x7E2F0CB0, + 0xC64D0D6E, 0x8F717049, 0x5435F720, 0x1D098A07, 0xE7508F03, 0xAE6CF224, 0x7528754D, 0x3C14086A, + 0x0D006599, 0x443C18BE, 0x9F789FD7, 0xD644E2F0, 0x2C1DE7F4, 0x65219AD3, 0xBE651DBA, 0xF759609D, + 0x4F3B6143, 0x06071C64, 0xDD439B0D, 0x947FE62A, 0x6E26E32E, 0x271A9E09, 0xFC5E1960, 0xB5626447, + 0x89766C2D, 0xC04A110A, 0x1B0E9663, 0x5232EB44, 0xA86BEE40, 0xE1579367, 0x3A13140E, 0x732F6929, + 0xCB4D68F7, 0x827115D0, 0x593592B9, 0x1009EF9E, 0xEA50EA9A, 0xA36C97BD, 0x782810D4, 0x31146DF3, + 0x1A00CB32, 0x533CB615, 0x8878317C, 0xC1444C5B, 0x3B1D495F, 0x72213478, 0xA965B311, 0xE059CE36, + 0x583BCFE8, 0x1107B2CF, 0xCA4335A6, 0x837F4881, 0x79264D85, 0x301A30A2, 0xEB5EB7CB, 0xA262CAEC, + 0x9E76C286, 0xD74ABFA1, 0x0C0E38C8, 0x453245EF, 0xBF6B40EB, 0xF6573DCC, 0x2D13BAA5, 0x642FC782, + 0xDC4DC65C, 0x9571BB7B, 0x4E353C12, 0x07094135, 0xFD504431, 0xB46C3916, 0x6F28BE7F, 0x2614C358, + 0x1700AEAB, 0x5E3CD38C, 0x857854E5, 0xCC4429C2, 0x361D2CC6, 0x7F2151E1, 0xA465D688, 0xED59ABAF, + 0x553BAA71, 0x1C07D756, 0xC743503F, 0x8E7F2D18, 0x7426281C, 0x3D1A553B, 0xE65ED252, 0xAF62AF75, + 0x9376A71F, 0xDA4ADA38, 0x010E5D51, 0x48322076, 0xB26B2572, 0xFB575855, 0x2013DF3C, 0x692FA21B, + 0xD14DA3C5, 0x9871DEE2, 0x4335598B, 0x0A0924AC, 0xF05021A8, 0xB96C5C8F, 0x6228DBE6, 0x2B14A6C1, + 0x34019664, 0x7D3DEB43, 0xA6796C2A, 0xEF45110D, 0x151C1409, 0x5C20692E, 0x8764EE47, 0xCE589360, + 0x763A92BE, 0x3F06EF99, 0xE44268F0, 0xAD7E15D7, 0x572710D3, 0x1E1B6DF4, 0xC55FEA9D, 0x8C6397BA, + 0xB0779FD0, 0xF94BE2F7, 0x220F659E, 0x6B3318B9, 0x916A1DBD, 0xD856609A, 0x0312E7F3, 0x4A2E9AD4, + 0xF24C9B0A, 0xBB70E62D, 0x60346144, 0x29081C63, 0xD3511967, 0x9A6D6440, 0x4129E329, 0x08159E0E, + 0x3901F3FD, 0x703D8EDA, 0xAB7909B3, 0xE2457494, 0x181C7190, 0x51200CB7, 0x8A648BDE, 0xC358F6F9, + 0x7B3AF727, 0x32068A00, 0xE9420D69, 0xA07E704E, 0x5A27754A, 0x131B086D, 0xC85F8F04, 0x8163F223, + 0xBD77FA49, 0xF44B876E, 0x2F0F0007, 0x66337D20, 0x9C6A7824, 0xD5560503, 0x0E12826A, 0x472EFF4D, + 0xFF4CFE93, 0xB67083B4, 0x6D3404DD, 0x240879FA, 0xDE517CFE, 0x976D01D9, 0x4C2986B0, 0x0515FB97, + 0x2E015D56, 0x673D2071, 0xBC79A718, 0xF545DA3F, 0x0F1CDF3B, 0x4620A21C, 0x9D642575, 0xD4585852, + 0x6C3A598C, 0x250624AB, 0xFE42A3C2, 0xB77EDEE5, 0x4D27DBE1, 0x041BA6C6, 0xDF5F21AF, 0x96635C88, + 0xAA7754E2, 0xE34B29C5, 0x380FAEAC, 0x7133D38B, 0x8B6AD68F, 0xC256ABA8, 0x19122CC1, 0x502E51E6, + 0xE84C5038, 0xA1702D1F, 0x7A34AA76, 0x3308D751, 0xC951D255, 0x806DAF72, 0x5B29281B, 0x1215553C, + 0x230138CF, 0x6A3D45E8, 0xB179C281, 0xF845BFA6, 0x021CBAA2, 0x4B20C785, 0x906440EC, 0xD9583DCB, + 0x613A3C15, 0x28064132, 0xF342C65B, 0xBA7EBB7C, 0x4027BE78, 0x091BC35F, 0xD25F4436, 0x9B633911, + 0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612, 0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F, + 0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8, 0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5 +}; + + +static +uint32_t _crc32c_sw(uint32_t crc, const void* data, size_t length) { + const char* p_buf = (const char*) data; + + // Handle leading misaligned bytes + size_t initial_bytes = (sizeof(int32_t) - (intptr_t)p_buf) & (sizeof(int32_t) - 1); + if (length < initial_bytes) initial_bytes = length; + for (size_t li = 0; li < initial_bytes; li++) { + crc = crc_tableil8_o32[(crc ^ *p_buf++) & 0x000000FF] ^ (crc >> 8); + } + + length -= initial_bytes; + size_t running_length = length & ~(sizeof(uint64_t) - 1); + size_t end_bytes = length - running_length; + + for (size_t li = 0; li < running_length/8; li++) { + crc ^= *(uint32_t*) p_buf; + p_buf += 4; + uint32_t term1 = crc_tableil8_o88[crc & 0x000000FF] ^ + crc_tableil8_o80[(crc >> 8) & 0x000000FF]; + uint32_t term2 = crc >> 16; + crc = term1 ^ + crc_tableil8_o72[term2 & 0x000000FF] ^ + crc_tableil8_o64[(term2 >> 8) & 0x000000FF]; + term1 = crc_tableil8_o56[(*(uint32_t *)p_buf) & 0x000000FF] ^ + crc_tableil8_o48[((*(uint32_t *)p_buf) >> 8) & 0x000000FF]; + + term2 = (*(uint32_t *)p_buf) >> 16; + crc = crc ^ term1 ^ + crc_tableil8_o40[term2 & 0x000000FF] ^ + crc_tableil8_o32[(term2 >> 8) & 0x000000FF]; + p_buf += 4; + } + + for (size_t li=0; li < end_bytes; li++) { + crc = crc_tableil8_o32[(crc ^ *p_buf++) & 0x000000FF] ^ (crc >> 8); + } + + return crc; +} + +#if HAS_INTEL_CPU +static inline +bool _has_hw_crc32c() { + + unsigned int ecx; + + asm ("movl $1, %%eax;" + "cpuid;" + "movl %%ecx, %0;" + : "=r"(ecx) // outputs + : // inputs + : "eax", "ebx", "ecx", "edx"); // clobber + + return (ecx & (1 << 20)) != 0; + +} + +#ifdef __WORDSIZE +#define BITS_PER_LONG __WORDSIZE +#else +#define BITS_PER_LONG 32 +#endif + +#if BITS_PER_LONG == 64 +#define REX_PRE "0x48, " +#define SCALE_F 8 +#else +#define REX_PRE +#define SCALE_F 4 +#endif + +static inline +uint32_t crc32c_intel_le_hw_byte(uint32_t crc, unsigned const char *data, + unsigned long length) { + while (length--) { + __asm__ __volatile__( + ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" + :"=S"(crc) + :"0"(crc), "c"(*data) + ); + data++; + } + + return crc; +} + +static inline +uint32_t _crc32c_intel(uint32_t crc, unsigned const char *data, unsigned long length) { + unsigned int iquotient = length / SCALE_F; + unsigned int iremainder = length % SCALE_F; + +#if BITS_PER_LONG == 64 + uint64_t *ptmp = (uint64_t *) data; +#else + uint32_t *ptmp = (uint32_t *) data; +#endif + + while (iquotient--) { + __asm__ __volatile__( + ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" + :"=S"(crc) + :"0"(crc), "c"(*ptmp) + ); + ptmp++; + } + + if (iremainder) + crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp, + iremainder); + + return crc; +} +#endif // HAS_INTEL_CPU + + +void bbcp_C32C::Init() { + C32Result = CRC32_XINIT; + TotLen = 0; + has_final = false; +#if HAS_INTEL_CPU + has_hw_crc32c = _has_hw_crc32c(); +#endif // HAS_INTEL_CPU +} + + +void bbcp_C32C::Update(const char *p, int reclen) +{ + +// Process each byte +// + TotLen += reclen; + C32Result = do_crc(C32Result, p, reclen); +} + +uint bbcp_C32C::do_crc(uint crc, const char *p, int reclen) +{ +#if HAS_INTEL_CPU + if (has_hw_crc32c) { + return _crc32c_intel(crc, (unsigned const char*)p, reclen); + } + return _crc32c_sw(crc, p, reclen); +#else + return _crc32c_sw(crc, p, reclen); +#endif +} diff --git a/src/bbcp_C32C.h b/src/bbcp_C32C.h new file mode 100644 index 0000000..a0f70c3 --- /dev/null +++ b/src/bbcp_C32C.h @@ -0,0 +1,114 @@ +#ifndef __BBCP_C32C_H__ +#define __BBCP_C32C_H__ +/******************************************************************************/ +/* */ +/* b b c p _ C 3 2 C . h */ +/* */ +/* ICRAR - International Centre for Radio Astronomy Research */ +/* (c) UWA - The University of Western Australia, 2016 */ +/* Copyright by UWA (in the framework of the ICRAR) */ +/* All rights reserved */ +/* */ +/* Produced by Rodrigo Tobar based on bbcp_C32.h */ +/* */ +/* bbcp is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* bbcp is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with bbcp in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include +#include +#include +#include + +#include "bbcp_ChkSum.h" +#include "bbcp_Endian.h" + + +/* Try to guess if we have an Intel CPU */ +#if defined __X86__ || defined __i386__ || defined i386 || defined _M_IX86 || defined __386__ || defined __x86_64__ || defined _M_X64 +#define HAS_INTEL_CPU 1 +#else +#define HAS_INTEL_CPU 0 +#endif + + +typedef unsigned int uint32; + +class bbcp_C32C : public bbcp_ChkSum +{ +public: + +void Init(); + +void Update(const char *Buff, int BLen); + +int csSize() { + return sizeof(C32Result); +} + + +char *csCurr(char **Text=0) { + if( has_final ) { + return (char *)&C32Result; + } + tmpcrc = finish(Text); + return (char *)&tmpcrc; + +} + +char *Final(char **Text=0) { + has_final = true; + C32Result = finish(Text); + return (char *)&C32Result; +} + +private: +uint finish(char **Text) + {char buff[sizeof(long long)]; + long long tLcs = TotLen; + int i = 0; + has_final = true; + uint crc = ~C32Result; +#ifndef BBCP_BIG_ENDIAN + crc = htonl(crc); +#endif + if (Text) *Text = x2a((char *)&crc); + return crc; + } + +public: +const char *Type() {return "c32c";} + + bbcp_C32C() {Init();} +virtual ~bbcp_C32C() {} + +private: +static const uint CRC32_XINIT = 0xffffffff; +static uint crctable[256]; + uint C32Result; + long long TotLen; + uint32_t tmpcrc; + bool has_final; +#if HAS_INTEL_CPU + bool has_hw_crc32c; +#endif + + uint do_crc(uint crc, const char *p, int reclen); + +}; +#endif diff --git a/src/bbcp_C32_Zip.C b/src/bbcp_C32_Zip.C new file mode 100644 index 0000000..e89de58 --- /dev/null +++ b/src/bbcp_C32_Zip.C @@ -0,0 +1,106 @@ +/******************************************************************************/ +/* */ +/* b b c p _ C 3 2 _ Z i p. C */ +/* */ +/* ICRAR - International Centre for Radio Astronomy Research */ +/* (c) UWA - The University of Western Australia, 2016 */ +/* Copyright by UWA (in the framework of the ICRAR) */ +/* All rights reserved */ +/* */ +/* Produced by Rodrigo Tobar */ +/* */ +/* bbcp is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* bbcp is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with bbcp in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include "bbcp_C32_Zip.h" + +static uint32_t crctable[] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, + 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, + 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, + 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, + 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, + 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, + 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, + 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, + 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, + 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, + 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, + 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, + 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, + 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, + 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, + 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, + 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, + 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, +}; + +void bbcp_C32_Zip::Update(const char *buf, int reclen) +{ + const uint8_t *p = reinterpret_cast(buf); + while(reclen--) { + crc = (crc >> 8) ^ crctable[ (crc ^ *p++) & 0xff]; + } +} diff --git a/src/bbcp_C32_Zip.h b/src/bbcp_C32_Zip.h new file mode 100644 index 0000000..c0d7bc3 --- /dev/null +++ b/src/bbcp_C32_Zip.h @@ -0,0 +1,97 @@ +#ifndef __BBCP_C32_ZIP_H__ +#define __BBCP_C32_ZIP_H__ +/******************************************************************************/ +/* */ +/* b b c p _ C 3 2 _ Z i p. h */ +/* */ +/* ICRAR - International Centre for Radio Astronomy Research */ +/* (c) UWA - The University of Western Australia, 2016 */ +/* Copyright by UWA (in the framework of the ICRAR) */ +/* All rights reserved */ +/* */ +/* Produced by Rodrigo Tobar based on bbcp_C32.h */ +/* */ +/* bbcp is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* bbcp is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with bbcp in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include +#include +#include + +#include "bbcp_ChkSum.h" +#include "bbcp_Endian.h" + +/** + * A PKZip-compatible CRC32 implementation. + * + * The CRC32 implementation available in the bbcp_C32 class is based in the CRC + * error checking described in the ISO/IEC 8802-3:1996 standard (Ethernet), and + * implemented for example by the cksum utility. This class follows instead the + * CRC32 algorithm used in the ISO 3309 standard, which is in turn used by the + * ZIP format as specified in RFC 1952, and by many other CRC32 implementors. + */ +class bbcp_C32_Zip : public bbcp_ChkSum +{ +public: + +void Init() { crc = CRC32_XINIT; has_final = false; } + +void Update(const char *Buff, int BLen); + +int csSize() {return sizeof(crc);} + +char *csCurr(char **Text=0) { + if( has_final ) { + return (char *)&crc; + } + tmpcrc = finish(Text); + return (char *)&tmpcrc; +} + +char *Final(char **Text=0) { + has_final = true; + crc = finish(Text); + return (char *)&crc; +} + +private: +uint32_t finish(char **Text=0) + { + uint32_t finished = crc ^ CRC32_XOROT; +#ifndef BBCP_BIG_ENDIAN + finished = htonl(finished); +#endif + if (Text) *Text = x2a((char *)&finished); + return finished; + } + +public: +const char *Type() {return "c32zip";} + + bbcp_C32_Zip() {Init();} +virtual ~bbcp_C32_Zip() {} + +private: +static const uint32_t CRC32_XINIT = 0xffffffff; +static const uint32_t CRC32_XOROT = 0xffffffff; + uint32_t crc; + uint32_t tmpcrc; + bool has_final; +}; +#endif diff --git a/src/bbcp_ChkSum.C b/src/bbcp_ChkSum.C index d9b86e2..3a72d79 100644 --- a/src/bbcp_ChkSum.C +++ b/src/bbcp_ChkSum.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ C h k S u m . C */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -25,11 +26,13 @@ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ -#include "bbcp_Config.h" #include "bbcp_ChkSum.h" +#include "bbcp_Config.h" #include "bbcp_A32.h" #include "bbcp_A32_zlib.h" #include "bbcp_C32.h" +#include "bbcp_C32_Zip.h" +#include "bbcp_C32C.h" #include "bbcp_MD5.h" #include "bbcp_MD5_openssl.h" @@ -49,6 +52,8 @@ bbcp_ChkSum *bbcp_ChkSum::Alloc(int csType) #else case bbcp_csMD5: return (bbcp_ChkSum *)new bbcp_MD5; #endif + case bbcp_csC32Z:return (bbcp_ChkSum *)new bbcp_C32_Zip; + case bbcp_csC32C:return (bbcp_ChkSum *)new bbcp_C32C; default: break; } diff --git a/src/bbcp_ChkSum.h b/src/bbcp_ChkSum.h index fdef7cf..886a689 100644 --- a/src/bbcp_ChkSum.h +++ b/src/bbcp_ChkSum.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ C h k S u m . h */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Config.C b/src/bbcp_Config.C index e8bc1a2..6ced0fd 100644 --- a/src/bbcp_Config.C +++ b/src/bbcp_Config.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ C o n f i g . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ @@ -84,7 +85,7 @@ /* G l o b a l O b j e c t s */ /******************************************************************************/ - bbcp_Config bbcp_Config; + bbcp_Config bbcp_Cfg; bbcp_Debug bbcp_Debug; @@ -118,7 +119,6 @@ bbcp_Config::bbcp_Config() SrcUser = 0; SrcHost = 0; SrcBlen = 0; - slkPath = 0; srcPath = 0; srcSpec = 0; srcLast = 0; @@ -135,6 +135,7 @@ bbcp_Config::bbcp_Config() bindtries = 1; bindwait = 0; Options = 0; + Option2 = 0; Mode = (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH) & ~uMask; ModeD = 0; ModeDC = (Mode |S_IXUSR|S_IXGRP|S_IXOTH) & ~uMask; @@ -164,8 +165,8 @@ bbcp_Config::bbcp_Config() Wsize = 131072; MaxWindow = 0; lastseqno = 0; - SrcArg = "SRC"; - SnkArg = "SNK"; + SrcArg = strdup("SRC"); + SnkArg = strdup("SNK"); CKPdir = 0; IDfn = 0; TimeLimit = 0; @@ -233,7 +234,10 @@ bbcp_Config::~bbcp_Config() #define Cat_Oct(x) { cbp=n2a(x,&cbp[0],"%o");} #define Add_Str(x) {cbp[0]=' '; strcpy(&cbp[1], x); cbp+=strlen(x)+1;} -#define bbcp_VALIDOPTS (char *)"-a.AB:b:C:c.d:DeE:fFghi:I:kKl:L:m:nN:oOpP:q:rR.s:S:t:T:u:U:vVw:W:x:y:zZ:4.~@:$#+" +#define bbcp_VALIDOPT1 (char *)"-a.AB:b:C:c.d:DeE:fFghi:I:kKl:L:m:nN:oOp" +#define bbcp_VALIDOPT2 "P:q:rR.s:S:t:T:u:U:vVw:W:x:y:zZ:4.~@:$#+" +#define bbcp_VALIDOPT3 "^<>" +#define bbcp_VALIDOPTS bbcp_VALIDOPT1 bbcp_VALIDOPT2 bbcp_VALIDOPT3 #define bbcp_SSOPTIONS bbcp_VALIDOPTS "MH:Y:" #define Hmsg1(a) {bbcp_Fmsg("Config", a); help(1);} @@ -255,6 +259,10 @@ void bbcp_Config::Arguments(int argc, char **argv, int cfgfd) if (argc < 2) Hmsg1("Copy arguments not specified."); notctl = (Options & (bbcp_SRC | bbcp_SNK)); +//DEBUG +// if (Options & bbcp_SRC) bbcp_HostName = "SRC"; +// if (Options & bbcp_SNK) bbcp_HostName = "SNK"; + // Establish valid options and the source of those options // if (notctl) arglist.Options(bbcp_SSOPTIONS, STDIN_FILENO); @@ -300,6 +308,7 @@ void bbcp_Config::Arguments(int argc, char **argv, int cfgfd) } else ParseSB(arglist.argval); break; case 'D': Options |= bbcp_TRACE; + chdir("/tmp"); // Where the core file goes break; case 'e': csOpts |= bbcp_csDashE|bbcp_csLink; break; @@ -310,7 +319,7 @@ void bbcp_Config::Arguments(int argc, char **argv, int cfgfd) break; case 'F': Options |= bbcp_NOSPCHK; break; - case 'g': Options |= bbcp_GROSS; + case 'g': // No longer needed so w ignore it for backward compatability break; case 'h': help(0); break; @@ -372,14 +381,18 @@ void bbcp_Config::Arguments(int argc, char **argv, int cfgfd) case 's': if (a2n("streams", arglist.argval, Streams,1,BBCP_MAXSTREAMS)) Cleanup(1, argv[0], cfgfd); break; - case 'S': if (SrcXeq) free(SrcXeq); - SrcXeq = strdup(arglist.argval); + case 'S': if (*arglist.argval != '+') + {if (SrcXeq) free(SrcXeq); + SrcXeq = strdup(arglist.argval); + } else setArgs("SRC", SrcArg, arglist.argval+1); break; case 't': if (a2sz("time limit", arglist.argval, TimeLimit,1,((int)1)<<30)) Cleanup(1, argv[0], cfgfd); break; - case 'T': if (SnkXeq) free(SnkXeq); - SnkXeq = strdup(arglist.argval); + case 'T': if (*arglist.argval != '+') + {if (SnkXeq) free(SnkXeq); + SnkXeq = strdup(arglist.argval); + } else setArgs("SNK", SnkArg, arglist.argval+1); break; case 'v': Options |= bbcp_VERBOSE; if (xTrace < 2) xTrace = 2; @@ -437,9 +450,15 @@ void bbcp_Config::Arguments(int argc, char **argv, int cfgfd) case '#': cout <': Option2 |= bbcp2_SKPOERR; + break; default: if (!notctl) {if (cfgfd < 0) help(255); Cleanup(1, argv[0], cfgfd); @@ -466,10 +485,6 @@ void bbcp_Config::Arguments(int argc, char **argv, int cfgfd) else MyHost = bbcp_Net.FullHostName((char *)0); bbcp_HostName = MyHost; -// Correct recursive readable selection option -// - if (Options & bbcp_GROSS) Options &= ~bbcp_RXONLY; - // If there is a checksum specification, process it now // if (csSpec && EOpts(csSpec)) Cleanup(1, argv[0], cfgfd); @@ -644,7 +659,7 @@ void bbcp_Config::Arguments(int argc, char **argv, int cfgfd) if (!CKPdir && Options & bbcp_APPEND && Options & bbcp_SNK) {const char *ckpsfx = "/.bbcp"; char *homedir = bbcp_OS.getHomeDir(); - CKPdir = (char *)malloc(strlen(homedir) + sizeof(ckpsfx) + 1); + CKPdir = (char *)malloc(strlen(homedir) + strlen(ckpsfx) + 1); strcpy(CKPdir, homedir); strcat(CKPdir, ckpsfx); if (mkdir(CKPdir, 0755) && errno != EEXIST) {bbcp_Emsg("Config",errno,"creating restart directory",CKPdir); @@ -701,10 +716,9 @@ H("-d path requests relative source path addressing and target path creation.") H("-D turns on debugging.") H("-e error check data for transmission errors using md5 checksum.") H("-E csa specify checksum alorithm and optionally report or verify checksum.") -H(" csa: [%]{a32|c32|md5}[=[ | ]]") +H(" csa: [%]{a32|c32|md5|c32z|c32c}[=[ | ]]") H("-f forces the copy by first unlinking the target file before copying.") H("-F does not check to see if there is enough space on the target node.") -H("-g do a gross copy (i.e. copy even if there are no directory entries).") H("-h print help information.") H("-i idfn is the name of the ssh identify file for source and target.") H("-I slfn is the name of the file that holds the list of files to be copied.") @@ -748,6 +762,7 @@ H("-@ specifies how symbolic links are handled: copy recreates the symlink, H(" follow copies the symlink target, and ignore skips it (default).") H("-$ print the license and exit.") H("-# print the version and exit.") +H("-^ omit copying empty directories (same as --omit-emptydirs).") H("-- allows an option with a defaulted optional arg to appear last.") I("user the user under which the copy is to be performed. The default is") H(" to use the current login name.") @@ -783,9 +798,9 @@ int bbcp_Config::ConfigInit(int argc, char **argv) // Make sure we have at least one argument to determine who we are // if (argc >= 2) - { if (!strcmp(argv[1], SrcArg)) + { if (!strcmp(argv[1], "SRC")) {Options |= bbcp_SRC; bbcp_Debug.Who = (char *)"SRC"; return 0;} - else if (!strcmp(argv[1], SnkArg)) + else if (!strcmp(argv[1], "SNK")) {Options |= bbcp_SNK; bbcp_Debug.Who = (char *)"SNK"; return 0;} else MyProg = strdup(argv[0]); } @@ -963,7 +978,6 @@ void bbcp_Config::Config_Ctl(int rwbsz) if (csOpts & bbcp_csDashE) Add_Opt('e'); if (Options & bbcp_FORCE) Add_Opt('f'); if (Options & bbcp_NOSPCHK) Add_Opt('F'); - if (Options & bbcp_GROSS) Add_Opt('g'); if (Options & bbcp_KEEP) Add_Opt('k'); if (Options & bbcp_NOUNLINK) Add_Opt('K'); if (LogSpec) {Add_Opt('L'); Add_Str(LogSpec);} @@ -979,6 +993,9 @@ void bbcp_Config::Config_Ctl(int rwbsz) if (Options & bbcp_XPIPE) {Add_Opt('N'); Add_Str(upSpec);} if (Options & bbcp_ORDER) Add_Opt('o'); if (Options & bbcp_OMIT) Add_Opt('O'); + if (Option2 & bbcp2_SKPEDIR) Add_Opt('^'); + if (Option2 & bbcp2_SKPIERR) Add_Opt('<'); + if (Option2 & bbcp2_SKPOERR) Add_Opt('>') if (Options & bbcp_PCOPY) Add_Opt((Options & bbcp_PTONLY ? '~' : 'p')); if (Progint) {Add_Opt('P'); Add_Num(Progint);} if ((n = bbcp_Net.QoS())) {Add_Opt('q'); Add_Num(n); } @@ -1009,7 +1026,7 @@ void bbcp_Config::Config_Ctl(int rwbsz) if (Options & bbcp_SLKEEP) {Add_Str("keep");} else {Add_Str("follow");} } - if (Options & (bbcp_RXONLY|bbcp_RDONLY)) Add_Opt('+'); + if (Options & bbcp_RDONLY) Add_Opt('+'); CopyOpts = strdup(cbuff); } @@ -1268,9 +1285,11 @@ int bbcp_Config::EOpts(char *opts) csLen = strlen(opts); if (csLen >= (int)sizeof(csName)) *csName = 0; else strcpy(csName, opts); - if (!strcmp("a32", csName)) {csType = bbcp_csA32; csSize = 4;} - else if (!strcmp("c32", csName)) {csType = bbcp_csC32; csSize = 4;} - else if (!strcmp("md5", csName)) {csType = bbcp_csMD5; csSize = 16;} + if (!strcmp("a32", csName)) {csType = bbcp_csA32; csSize = 4;} + else if (!strcmp("c32", csName)) {csType = bbcp_csC32; csSize = 4;} + else if (!strcmp("c32z", csName)) {csType = bbcp_csC32Z; csSize = 4;} + else if (!strcmp("c32c", csName)) {csType = bbcp_csC32C; csSize = 4;} + else if (!strcmp("md5", csName)) {csType = bbcp_csMD5; csSize = 16;} else {bbcp_Fmsg("Config", "Invalid checksum type -", opts); return -1;} // Verify the checksum value if one was actually specified @@ -1424,6 +1443,8 @@ int bbcp_Config::ROpts(char *opts) || (rtLimit = atoi(rOpts+2)) <= 0) return ROptsErr(rOpts); break; + case 't': Options |= bbcp_RTTMP; + break; case 'v': Options |= bbcp_RTCVERC; break; case '/': @@ -1449,6 +1470,32 @@ int bbcp_Config::ROptsErr(char *eTxt) return -1; } +/******************************************************************************/ +/* s e t A r g s */ +/******************************************************************************/ + +void bbcp_Config::setArgs(const char *aType, char *&aDst, char *aSrc) +{ + int n; + +// Free up whatever is in the destination arg +// + if (aDst) free(aDst); + +// Trim of leading spaces from the source arg +// + while(*aSrc == ' ') aSrc++; + +// Allocate needed storage +// + n = strlen(aType) + 1 + strlen(aSrc) + 1; + aDst = (char *)malloc(n); + +// Format the arguments and set them +// + snprintf(aDst, n, "%s %s", aType, aSrc); +} + /******************************************************************************/ /* s e t C S */ /******************************************************************************/ @@ -1497,48 +1544,58 @@ void bbcp_Config::setOpts(bbcp_Args &Args) Args.Option("config", 6, 'C', ':'); Args.Option("dirbase", 3, 'd', ':'); Args.Option("debug", 5, 'D', 0); -// e +// e for -e (same as --checksum) Args.Option("checksum", 5, 'E', ':'); Args.Option("force", 1, 'f', 0); Args.Option("nofschk", 4, 'F', ':'); - Args.Option("gross", 1, 'g', 0); + Args.Option("gross", 1, 'g', 0); // deprecated and ignored +// G is available Args.Option("help", 1, 'h', ':'); Args.Option("idfile", 1, 'i', ':'); Args.Option("infiles", 2, 'I', ':'); Args.Option("ipv4", 4, '4', '.'); Args.Option("keep", 1, 'k', 0); -// K +// K for -K Args.Option("license", 7, '$', 0); Args.Option("links", 4, '@', ':'); // synonym for --symlinks Args.Option("logfile", 1, 'l', ':'); -// L +// L for -L Args.Option("mkdir", 3, 'A', 0); Args.Option("mode", 1, 'm', ':'); Args.Option("nodns", 1, 'n', 0); Args.Option("pipe", 4, 'N', ':'); Args.Option("order", 1, 'o', 0); - Args.Option("omit", 4, 'O', 0); + Args.Option("omit", 4, 'O', 0); // List in increasing min length! + Args.Option("omit-ed", 7, '^', 0); + Args.Option("omit-ie", 7, '<', 0); + Args.Option("omit-oe", 7, '>', 0); + Args.Option("omit-dups", 9, 'O', 0); + Args.Option("omit-emptydirs", 11, '^', 0); + Args.Option("omit-inerrs", 11, '<', 0); + Args.Option("omit-outerrs", 12, '>', 0); Args.Option("preserve", 1, 'p', 0); Args.Option("progress", 4, 'P', ':'); Args.Option("ptime", 2, '~', 0); Args.Option("qos", 3, 'q', ':'); +// Q is available Args.Option("readable", 4, '+', 0); Args.Option("realtime", 4, 'R', ':'); Args.Option("recursive", 1, 'r', 0); Args.Option("streams", 1, 's', ':'); Args.Option("symlinks", 7, '@', ':'); // synonym for --links -// S +// S for -S Args.Option("timelimit", 1, 't', ':'); -// T +// T for -T Args.Option("verbose", 1, 'v', 0); Args.Option("vverbose", 2, 'V', 0); Args.Option("version", 7, '#', 0); Args.Option("unbuffered", 1, 'u', ':'); -// U +// U for -U Args.Option("windowsz", 1, 'w', ':'); Args.Option("xfrrate", 1, 'x', ':'); -// Y +// Y used for password Args.Option("sync", 4, 'y', 0); +// X is available Args.Option("reverse", 3, 'z', 0); Args.Option("port", 4, 'Z', ':'); } diff --git a/src/bbcp_Config.h b/src/bbcp_Config.h index ed6877e..fc9e1bc 100644 --- a/src/bbcp_Config.h +++ b/src/bbcp_Config.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ C o n f i g . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ @@ -39,6 +40,7 @@ class bbcp_Config public: long long Options; +long long Option2; int bindtries; int bindwait; int Mode; @@ -109,6 +111,8 @@ int csType; #define bbcp_csA32 1 #define bbcp_csC32 2 #define bbcp_csMD5 3 +#define bbcp_csC32Z 4 +#define bbcp_csC32C 5 int csFD; char *csPath; @@ -120,14 +124,13 @@ char csString[44]; char ubSpec[4]; char upSpec[4]; -bbcp_FileSpec *slkPath; bbcp_FileSpec *srcPath; bbcp_FileSpec *srcSpec; bbcp_FileSpec *srcLast; bbcp_FileSpec *snkSpec; -const char *SrcArg; -const char *SnkArg; +char *SrcArg; +char *SnkArg; void Arguments(int argc, char **argv, int cfgfd=-1); int ConfigInit(int argc, char **argv); @@ -164,16 +167,18 @@ void ParseSB(char *spec); int ROpts(char *Opts); int ROptsErr(char *Opts); char *Rtoken(void); +void setArgs(const char *aType, char *&aDst, char *aSrc); int setIPV4(char *Opts); void setOpts(bbcp_Args &Args); int setPorts(char *pspec); +bool setSkip(char *skparg); char *tohex(char *inbuff, int inlen, char *outbuff); int Unbuff(char *Opts); int Unpipe(char *Opts); }; /******************************************************************************/ -/* O p t i o n D e f i n o i t i o n s */ +/* O p t i o n s B i t D e f i n i t i o n s */ /******************************************************************************/ #define bbcp_APPEND 0x0000000000000001LL @@ -239,11 +244,24 @@ int Unpipe(char *Opts); #define bbcp_AUTOMKD 0x0001000000000000LL #define bbcp_SLFOLLOW 0x0002000000000000LL #define bbcp_SLKEEP 0x0004000000000000LL -#define bbcp_GROSS 0x0008000000000000LL + #define bbcp_PTONLY 0x0010000000000000LL #define bbcp_RDONLY 0x0020000000000000LL -#define bbcp_RXONLY 0x0040000000000000LL +// bbcp_??? 0x0040000000000000LL +#define bbcp_RTTMP 0x0080000000000000LL + +/******************************************************************************/ +/* O p t i o n 2 B i t D e f i n i t i o n s */ +/******************************************************************************/ + +#define bbcp2_SKPEDIR 0x0000000000000001LL +#define bbcp2_SKPIERR 0x0000000000000002LL +#define bbcp2_SKPOERR 0x0000000000000004LL +/******************************************************************************/ +/* M i s c V a l u e s */ +/******************************************************************************/ + #define BBCP_MAXSTREAMS 64 #define BBCP_MINPMONSEC 1 diff --git a/src/bbcp_Debug.h b/src/bbcp_Debug.h index 1c40e29..cd8709c 100644 --- a/src/bbcp_Debug.h +++ b/src/bbcp_Debug.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ D e b u g . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Emsg.C b/src/bbcp_Emsg.C index 298a38d..d91b5a3 100644 --- a/src/bbcp_Emsg.C +++ b/src/bbcp_Emsg.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ E m s g . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Emsg.h b/src/bbcp_Emsg.h index 821798d..5cc6bd5 100644 --- a/src/bbcp_Emsg.h +++ b/src/bbcp_Emsg.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ E m s g . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Endian.h b/src/bbcp_Endian.h index 90b95a7..8ee6da5 100644 --- a/src/bbcp_Endian.h +++ b/src/bbcp_Endian.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ E n d i a n . h */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_FS_Null.C b/src/bbcp_FS_Null.C index 1b25810..a3ac9f5 100644 --- a/src/bbcp_FS_Null.C +++ b/src/bbcp_FS_Null.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ F S _ N u l l . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_FS_Null.h b/src/bbcp_FS_Null.h index 605d836..ee238f0 100644 --- a/src/bbcp_FS_Null.h +++ b/src/bbcp_FS_Null.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ F S _ N u l l . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_FS_Pipe.C b/src/bbcp_FS_Pipe.C index 64bb79a..ee5a098 100644 --- a/src/bbcp_FS_Pipe.C +++ b/src/bbcp_FS_Pipe.C @@ -2,8 +2,9 @@ /* */ /* b b c p _ F S _ P i p e . C */ /* */ -/* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ -/* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2011-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ +/* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_FS_Pipe.h b/src/bbcp_FS_Pipe.h index aa105fc..1598e61 100644 --- a/src/bbcp_FS_Pipe.h +++ b/src/bbcp_FS_Pipe.h @@ -4,8 +4,9 @@ /* */ /* b b c p _ F S _ P i p e . h */ /* */ -/* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ -/* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2011-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ +/* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_FS_Unix.C b/src/bbcp_FS_Unix.C index 6369729..a71379e 100644 --- a/src/bbcp_FS_Unix.C +++ b/src/bbcp_FS_Unix.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ F S _ U n i x . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -33,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -74,28 +76,35 @@ extern bbcp_System bbcp_OS; int bbcp_FS_Unix::Applicable(const char *path) { +// Duplicate the path and only consider the first component (mount point) +// + if (!fs_path) + {char *slash; + fs_path = strdup(path); + if ((slash = index(fs_path+1, '/'))) *slash = 0; + } + +// FREEBSD doesn't have a statvfs, so we punt +// #ifdef FREEBSD - if (!fs_path) fs_path = strdup(path); + secSize = 512; #else struct statvfs buf; // To find out whether or not we are applicable, simply do a statvfs on the // incomming path. If we can do it, then we are a unix filesystem. // - if (statvfs(path, &buf)) return 0; + if (statvfs(fs_path, &buf)) return 0; // Set the sector size // secSize = buf.f_frsize; -// Save the path to this filesystem if we don't have one. This is a real -// kludgy short-cut since in bbcp we only have a single output destination. +// Record the unique filesystem ID. // - if (!fs_path) - {fs_path = strdup(path); - memcpy((void *)&fs_id, (const void *)&buf.f_fsid, sizeof(fs_id)); - } + memcpy((void *)&fs_id, (const void *)&buf.f_fsid, sizeof(fs_id)); #endif + return 1; } @@ -181,9 +190,6 @@ long long bbcp_FS_Unix::getSize(int fd, long long *bsz) int bbcp_FS_Unix::MKDir(const char *path, mode_t mode) { if (mkdir(path, mode)) return -errno; - - if (chmod(path, 0755)) return -errno; - return 0; } diff --git a/src/bbcp_FS_Unix.h b/src/bbcp_FS_Unix.h index 8b1fc27..de0dc8c 100644 --- a/src/bbcp_FS_Unix.h +++ b/src/bbcp_FS_Unix.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ F S _ U n i x . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_File.C b/src/bbcp_File.C index cbe1bb3..7ffca45 100644 --- a/src/bbcp_File.C +++ b/src/bbcp_File.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ F i l e . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -74,7 +75,7 @@ extern bbcp_BuffPool bbcp_BPool; extern bbcp_BuffPool bbcp_CPool; -extern bbcp_Config bbcp_Config; +extern bbcp_Config bbcp_Cfg; extern bbcp_RTCopy bbcp_RTCopy; @@ -100,7 +101,7 @@ void *bbcp_FileCSX(void *pp) // Send checksum to the target if so needed // - if ((bbcp_Config.csOpts & bbcp_csSend) && bP && bP->boff >= 0) + if ((bbcp_Cfg.csOpts & bbcp_csSend) && bP && bP->boff >= 0) {bP->blen = -BBCP_CLCKS; memcpy(bP->bHdr.cksm, csP->csObj->csCurr(), csLen); } @@ -138,7 +139,7 @@ bbcp_File::bbcp_File(const char *path, bbcp_IO *iox, bufreorders = 0; maxreorders = 0; PaceTime = 0; - rtCopy = (bbcp_Config.Options & bbcp_RTCSRC ? 1 : 0); + rtCopy = (bbcp_Cfg.Options & bbcp_RTCSRC ? 1 : 0); } /******************************************************************************/ @@ -178,7 +179,7 @@ int bbcp_File::Passthru(bbcp_BuffPool *iBP, bbcp_BuffPool *oBP, bbcp_ChkSum *csObj; long long Offset = nextoffset; int csLen, csVer, numadd, maxbufs, maxadds = nstrms; - int rc = 0, unordered = !(bbcp_Config.Options & bbcp_ORDER); + int rc = 0, unordered = !(bbcp_Cfg.Options & bbcp_ORDER); // Determine if we will be piggy-backing checksumming here // @@ -300,9 +301,9 @@ int bbcp_File::Read_All(bbcp_BuffPool &inPool, int Vn) // Set up checksumming. We would prefer to do this in the calling thread but // this is easier. One day we will generalize buffer piping. // - if (bbcp_Config.csOpts & bbcp_csVerIn) - {csP = new bbcp_FileChkSum(&inPool, this, bbcp_Config.csType, - bbcp_Config.csOpts & bbcp_csVerIO); + if (bbcp_Cfg.csOpts & bbcp_csVerIn) + {csP = new bbcp_FileChkSum(&inPool, this, bbcp_Cfg.csType, + bbcp_Cfg.csOpts & bbcp_csVerIO); if ((rc = bbcp_Thread_Start(bbcp_FileCSX, (void *)csP, &tid)) < 0) {bbcp_Emsg("File", rc, "starting file checksum thread."); delete csP; @@ -314,12 +315,12 @@ int bbcp_File::Read_All(bbcp_BuffPool &inPool, int Vn) // Establish logging options // - if (bbcp_Config.Options & bbcp_LOGRD) IOB->Log("DISK", 0); + if (bbcp_Cfg.Options & bbcp_LOGRD) IOB->Log("DISK", 0); // Determine what kind of reading we will do here and do it // // cerr <<"BLOCKSIZE " < 1 ? Read_Vector(&inPool, outPool, Vn) @@ -332,7 +333,7 @@ int bbcp_File::Read_All(bbcp_BuffPool &inPool, int Vn) // Check if we ended because with an error // if (rc && rc != -ENOBUFS) - {const char *Act=(bbcp_Config.Options & bbcp_XPIPE ? "piping":"writing"); + {const char *Act=(bbcp_Cfg.Options & bbcp_XPIPE ? "piping":"writing"); bbcp_Emsg("Read", -rc, Act, iofn); } @@ -354,10 +355,10 @@ int bbcp_File::Read_All(bbcp_BuffPool &inPool, int Vn) if (csP) {csP->csPool.putFullBuff(bP); bbcp_Thread_Wait(tid); - if (!rc && *bbcp_Config.csString) + if (!rc && *bbcp_Cfg.csString) {char *csTxt, *csVal = csP->csObj->Final(&csTxt); - if (memcmp(csVal, bbcp_Config.csValue, csP->csObj->csSize())) - {bbcp_Fmsg("Read", iofn, "source checksum", bbcp_Config.csString, + if (memcmp(csVal, bbcp_Cfg.csValue, csP->csObj->csSize())) + {bbcp_Fmsg("Read", iofn, "source checksum", bbcp_Cfg.csString, "does not match", csTxt); rc = EILSEQ; } else {DEBUG(csP->csObj->Type() <<": " <(bbcp_Config.Xrate) + nBlk = static_cast(bbcp_Cfg.Xrate) / static_cast(rdsz); PaceTime = static_cast(usPerSec / nBlk); DEBUG("Pacing " <csObj->csSize(); char *csTxt, *csVal = csP->csObj->Final(&csTxt); - if (*bbcp_Config.csString) - {if (memcmp(csVal, bbcp_Config.csValue, csP->csObj->csSize())) - {bbcp_Fmsg("Write", iofn, "target checksum", bbcp_Config.csString, + if (*bbcp_Cfg.csString) + {if (memcmp(csVal, bbcp_Cfg.csValue, csP->csObj->csSize())) + {bbcp_Fmsg("Write", iofn, "target checksum", bbcp_Cfg.csString, "does not match", csTxt); return -EILSEQ; } } else { - memcpy(bbcp_Config.csValue, csVal, csLen); - strcpy(bbcp_Config.csString, csTxt); + memcpy(bbcp_Cfg.csValue, csVal, csLen); + strcpy(bbcp_Cfg.csString, csTxt); } } // All done // - DEBUG(csP->csObj->Type() <<": '" <csObj->Type() <<": '" <Log(0, "DISK"); + if (bbcp_Cfg.Options & bbcp_LOGRD) IOB->Log(0, "DISK"); // Determine what kind of writing we will do here and do it // @@ -693,7 +694,7 @@ int bbcp_File::Write_All(bbcp_BuffPool &inPool, int nstrms) // Check if we ended because of an error or end of file // if (rc < 0 && rc != -ENOBUFS) - {const char *Act=(bbcp_Config.Options & bbcp_XPIPE ? "piping":"writing"); + {const char *Act=(bbcp_Cfg.Options & bbcp_XPIPE ? "piping":"writing"); bbcp_Emsg("Write", -rc, Act, iofn); inPool.Abort(); } @@ -708,13 +709,13 @@ int bbcp_File::Write_All(bbcp_BuffPool &inPool, int nstrms) // If checksums are being printed, send off ours if we have it // - if (bbcp_Config.csOpts & bbcp_csPrint && *bbcp_Config.csString) - cout <<"200 cks: " <Fsync((bbcp_Config.Options & bbcp_DSYNC ? iofn:0),IOB->FD()))) + if (!rc && IOB && (bbcp_Cfg.Options & bbcp_FSYNC) + && (rc = FSp->Fsync((bbcp_Cfg.Options & bbcp_DSYNC ? iofn:0),IOB->FD()))) bbcp_Emsg("Write", -rc, "synchronizing", iofn); // Close the output file and make sure it's ok diff --git a/src/bbcp_File.h b/src/bbcp_File.h index 507c3eb..6cb387d 100644 --- a/src/bbcp_File.h +++ b/src/bbcp_File.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ F i l e . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_FileSpec.C b/src/bbcp_FileSpec.C index 1351462..4eca966 100644 --- a/src/bbcp_FileSpec.C +++ b/src/bbcp_FileSpec.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ F i l e S p e c . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -46,7 +47,7 @@ /* E x t e r n a l O b j e c t s */ /******************************************************************************/ -extern bbcp_Config bbcp_Config; +extern bbcp_Config bbcp_Cfg; /******************************************************************************/ /* L o c a l D e f i n i t i o n s */ @@ -78,25 +79,25 @@ extern "C" void *bbcp_FileSpecIndex(void *pp) { static int negVal = -1; - bbcp_FileSpec *fP = bbcp_Config.srcSpec; - char xBuff[128]; - time_t tNow, xMsg = time(0)+bbcp_Config.Progint; - int numD = 0, numF = 0, numL = 0, slOpt; - int Blab = (bbcp_Config.Options & bbcp_VERBOSE) || bbcp_Config.Progint; + bbcp_FileSpec *fP = bbcp_Cfg.srcSpec; + char xBuff[256], yBuff[128]; + time_t tNow, xMsg = time(0)+bbcp_Cfg.Progint; + int numD = 0, numE = 0, numF = 0, numL = 0, slOpt; + int Blab = (bbcp_Cfg.Options & bbcp_VERBOSE) || bbcp_Cfg.Progint; bool aOK = true; // If we are following symlinks then we need to allocate an unordered set // to keep track of them to prevent copying self-referential trees. In any // case, establish the symlink processing option. // - if (bbcp_Config.Options & bbcp_SLFOLLOW) + if (bbcp_Cfg.Options & bbcp_SLFOLLOW) {slOpt = 0; pathSet = new bbcp_Set();} - else if (bbcp_Config.Options & bbcp_SLKEEP) slOpt = -1; + else if (bbcp_Cfg.Options & bbcp_SLKEEP) slOpt = -1; else slOpt = 1; // Set lastp to the last node in the list // - for (lastp = bbcp_Config.srcSpec; lastp->next != NULL; lastp = lastp->next) + for (lastp = bbcp_Cfg.srcSpec; lastp->next != NULL; lastp = lastp->next) ; // nothing to do -- just setting lastp // Extend all directories with the files therein @@ -104,15 +105,21 @@ void *bbcp_FileSpecIndex(void *pp) if (Blab) bbcp_Fmsg("Dirlist", "Indexing files to be copied..."); while(fP && aOK) {if ('d' == fP->Info.Otype) - {numD++; aOK = fP->ExtendFileSpec(numF, numL, slOpt);} + {numD++; aOK = fP->ExtendFileSpec(numF, numL, slOpt); + if (fP->isEmpty) + {numE++; + if (!(bbcp_Cfg.Option2 & bbcp2_SKPEDIR)) fP->isEmpty = false; + } + } fP = fP->next; - if (bbcp_Config.Progint && (tNow = time(0)) >= xMsg) - {sprintf(xBuff, "%d file%s and %d link%s in %d director%s so far...", + if (bbcp_Cfg.Progint && (tNow = time(0)) >= xMsg) + {sprintf(xBuff, "%d file%s and %d link%s in %d director%s " + "(%d empty) so far...", numF, (numF == 1 ? "" : "s"), numL, (numL == 1 ? "" : "s"), - numD, (numD == 1 ? "y": "ies")); + numD, (numD == 1 ? "y": "ies"), numE); bbcp_Fmsg("Dirlist", "Found", xBuff); - xMsg = tNow+bbcp_Config.Progint; + xMsg = tNow+bbcp_Cfg.Progint; } } @@ -130,10 +137,21 @@ void *bbcp_FileSpecIndex(void *pp) // Indicate what we found if so wanted // if (Blab) - {sprintf(xBuff, "%d file%s and %d link%s in %d director%s.", + {if (numE) + {numD -= numE; + if (bbcp_Cfg.Option2 & bbcp2_SKPEDIR) + {sprintf(yBuff, " with %d empty director%s skipped", + numE, (numE == 1 ? "y": "ies")); + } else { + sprintf(yBuff, " plus %d empty director%s", + numE, (numE == 1 ? "y": "ies")); + } + } else *yBuff = 0; + + sprintf(xBuff, "%d file%s and %d link%s in %d director%s%s.", numF, (numF == 1 ? "" : "s"), numL, (numL == 1 ? "" : "s"), - numD, (numD == 1 ? "y": "ies")); + numD, (numD == 1 ? "y": "ies"), yBuff); bbcp_Fmsg("Source", "Copying", xBuff); } @@ -173,7 +191,7 @@ int bbcp_FileSpec::Compose(long long did, char *dpath, int dplen, char *fname) // Get the current state of the file or directory // if ((retc = FSp->Stat(targpath, &Targ))) targetsz = 0; - else if (Targ.Otype == 'p' && (bbcp_Config.Options & bbcp_XPIPE)) + else if (Targ.Otype == 'p' && (bbcp_Cfg.Options & bbcp_XPIPE)) {targetsz = 0; return 0;} else if (Targ.Otype != 'f') {targetsz = -1; return 0;} else {targetsz = (long long)Targ.size; @@ -183,12 +201,12 @@ int bbcp_FileSpec::Compose(long long did, char *dpath, int dplen, char *fname) // Create signature filename if append mode is enabled and this is a file // - if (bbcp_Config.Options & bbcp_APPEND) + if (bbcp_Cfg.Options & bbcp_APPEND) {char buff[1025], *rp; if ((rp = rindex(targetfn,'/'))) rp++; else rp = targetfn; snprintf(buff, sizeof(buff)-1, "%s/bbcp.%s.%llx.%s", - bbcp_Config.CKPdir, hostname, did, rp); + bbcp_Cfg.CKPdir, hostname, did, rp); buff[sizeof(buff)-1] = '\0'; targsigf = strdup(buff); DEBUG("Append signature file is " <MKDir(targpath, bbcp_Config.ModeDC))) + if ((retc = FSp->MKDir(targpath, bbcp_Cfg.ModeDC))) {if (retc == -EEXIST) return 0; else return bbcp_Emsg("Create_Path", retc, "creating path", targpath); } @@ -367,10 +385,10 @@ bool bbcp_FileSpec::ExtendFileSpec(int &numF, int &numL, int slOpt) DIR *dirp; char relative_name[1024], absolute_name[4096]; struct stat sbuf; - int accD = (bbcp_Config.Options & bbcp_RXONLY ? R_OK|X_OK : 0); - int accF = (bbcp_Config.Options & bbcp_RDONLY ? R_OK : 0); - int dirFD; - bool aOK = true, blab = (bbcp_Config.Options & bbcp_VERBOSE) != 0; + int accD = (bbcp_Cfg.Options & bbcp_RDONLY ? R_OK|X_OK : 0); + int accF = (bbcp_Cfg.Options & bbcp_RDONLY ? R_OK : 0); + int dirFD, numEnt = 0; + bool aOK = true, blab = (bbcp_Cfg.Options & bbcp_VERBOSE) != 0; // Open the directory as we will need a file descriptor to it. Different // operaing systems have different ways of doing this. @@ -453,8 +471,13 @@ bool bbcp_FileSpec::ExtendFileSpec(int &numF, int &numL, int slOpt) newp->next = NULL; lastp->next = newp; lastp = newp; + numEnt++; } + // Flag this directory as empty or not. + // + if (!numEnt) isEmpty = true; + if (dirp) closedir(dirp); // This also closes the underlying fd return aOK; } @@ -469,11 +492,11 @@ int bbcp_FileSpec::Finalize(int retc) // If an error occured, see what we should do // if (retc) - {if (bbcp_Config.Options & (bbcp_KEEP|bbcp_NOUNLINK)) return retc; + {if (bbcp_Cfg.Options & (bbcp_KEEP|bbcp_NOUNLINK)) return retc; FSp->RM(targpath); } - else if (bbcp_Config.Options & bbcp_PCOPY) setStat(bbcp_Config.Mode); - else FSp->setMode(targpath, bbcp_Config.Mode); + else if (bbcp_Cfg.Options & bbcp_PCOPY) setStat(bbcp_Cfg.Mode); + else FSp->setMode(targpath, bbcp_Cfg.Mode); // Delete the signature file if one exists // @@ -498,7 +521,7 @@ void bbcp_FileSpec::Parse(char *spec, int isPipe) if (fspec) free(fspec); fspec = strdup(spec); username = hostname = pathname = filename = filereqn = 0; - seqno = bbcp_Config.lastseqno++; + seqno = bbcp_Cfg.lastseqno++; // Prepare to parse the spec // @@ -538,15 +561,15 @@ void bbcp_FileSpec::Parse(char *spec, int isPipe) filereqn = filename; } else { filename = filereqn = pathname; - bbcp_Config.Options |= bbcp_RELATIVE; - if (!username) username = bbcp_Config.SrcUser; - if (!hostname) hostname = bbcp_Config.SrcHost; - if (bbcp_Config.Options & bbcp_SRC && bbcp_Config.SrcBase) - {fspec1 = (char *)malloc(strlen(pathname)+bbcp_Config.SrcBlen+1); - strcpy(fspec1, bbcp_Config.SrcBase); - strcpy(fspec1+bbcp_Config.SrcBlen, pathname); + bbcp_Cfg.Options |= bbcp_RELATIVE; + if (!username) username = bbcp_Cfg.SrcUser; + if (!hostname) hostname = bbcp_Cfg.SrcHost; + if (bbcp_Cfg.Options & bbcp_SRC && bbcp_Cfg.SrcBase) + {fspec1 = (char *)malloc(strlen(pathname)+bbcp_Cfg.SrcBlen+1); + strcpy(fspec1, bbcp_Cfg.SrcBase); + strcpy(fspec1+bbcp_Cfg.SrcBlen, pathname); pathname = fspec1; - filename = filereqn = fspec1+bbcp_Config.SrcBlen; + filename = filereqn = fspec1+bbcp_Cfg.SrcBlen; BuildPaths(); } } @@ -591,13 +614,13 @@ int bbcp_FileSpec::setStat(mode_t Mode) // Set the mode (mode depends on whether this is a plain preserve or not) // - if (!(bbcp_Config.Options & bbcp_PTONLY)) Mode = Info.mode; + if (!(bbcp_Cfg.Options & bbcp_PTONLY)) Mode = Info.mode; if ((retc = FSp->setMode (targpath, Mode))) {act = (char *)"setting mode on"; ecode = retc;} // Set the group only if this is a plain preserve // - if (!(bbcp_Config.Options & bbcp_PTONLY) && Info.Group) + if (!(bbcp_Cfg.Options & bbcp_PTONLY) && Info.Group) FSp->setGroup (targpath, Info.Group); // Check if any errors occured (we ignore these just like cp/scp does) @@ -649,7 +672,7 @@ int bbcp_FileSpec::Stat(int complain) // filespec object only becomes valid after the first stat() call). // if (!FSp) - {fsOpts = (bbcp_Config.Options&bbcp_XPIPE ? bbcp_FileSystem::getFS_Pipe:0); + {fsOpts = (bbcp_Cfg.Options&bbcp_XPIPE ? bbcp_FileSystem::getFS_Pipe:0); if (!(FSp = bbcp_FileSystem::getFS(pathname, fsOpts))) {char savefn = *filename; *filename = '\0'; @@ -708,16 +731,16 @@ int bbcp_FileSpec::WriteSigFile() int bbcp_FileSpec::Xfr_Done() { struct stat sbuff; - int rc, Force = bbcp_Config.Options & bbcp_FORCE; + int rc, Force = bbcp_Cfg.Options & bbcp_FORCE; // Check if the output was a pipe // - if (bbcp_Config.Options & bbcp_OPIPE) {targetsz = 0; return 0;} + if (bbcp_Cfg.Options & bbcp_OPIPE) {targetsz = 0; return 0;} // If this is an APPEND request, build the signature file // //cerr <<"tsz=" <next = PS_Cur->next; PS_Cur->next = PS_New;} else if (PS_Prv) PS_Prv->next = PS_New; - else bbcp_Config.srcPath = PS_New; + else bbcp_Cfg.srcPath = PS_New; PS_Prv = PS_New; PS_Cur = PS_New->next; } if ((*cp = delim)) cp++; @@ -845,7 +868,7 @@ int bbcp_FileSpec::Xfr_Fixup() // Inform the person we will try to complete the copy // - if (bbcp_Config.Options & bbcp_VERBOSE) + if (bbcp_Cfg.Options & bbcp_VERBOSE) bbcp_Fmsg("Xfr_Fixup", "Will try to complete copying",targpath); return 0; } diff --git a/src/bbcp_FileSpec.h b/src/bbcp_FileSpec.h index 2a40e04..ac9ecb0 100644 --- a/src/bbcp_FileSpec.h +++ b/src/bbcp_FileSpec.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ F i l e S p e c . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -50,6 +51,7 @@ char *targetfn; long long targetsz; char *targsigf; int seqno; +bool isEmpty; struct bbcp_FileInfo Info; int Compose(long long did, char *dpath, int dplen, char *fname); @@ -86,6 +88,7 @@ int Xfr_Done(); : next(0), username(uname), hostname(hname), pathname(0), filename(0), filereqn(0), fileargs(0), targpath(0), targetfn(0), targetsz(0), targsigf(0), + isEmpty(false), fspec(0), fspec1(0), fspec2(0), slData(0), FSp(fsp) {} ~bbcp_FileSpec() {if (fspec) free(fspec); if (fspec1) free(fspec1); diff --git a/src/bbcp_FileSystem.C b/src/bbcp_FileSystem.C index 1a203de..3b53673 100644 --- a/src/bbcp_FileSystem.C +++ b/src/bbcp_FileSystem.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ F i l e S y s t e m . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_FileSystem.h b/src/bbcp_FileSystem.h index 41cd86b..e226d32 100644 --- a/src/bbcp_FileSystem.h +++ b/src/bbcp_FileSystem.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ F i l e S y s t e m . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Headers.h b/src/bbcp_Headers.h index fc890fb..ff72221 100644 --- a/src/bbcp_Headers.h +++ b/src/bbcp_Headers.h @@ -4,7 +4,7 @@ /* */ /* b b c p _ H e a d e r s . h */ /* */ -/* (c) 2009 by the Board of Trustees of the Leland Stanford, Jr., University */ +/*(c) 2009-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ diff --git a/src/bbcp_IO.C b/src/bbcp_IO.C index 356afb3..3d02d44 100644 --- a/src/bbcp_IO.C +++ b/src/bbcp_IO.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ I O . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_IO.h b/src/bbcp_IO.h index cebb1e4..e19836a 100644 --- a/src/bbcp_IO.h +++ b/src/bbcp_IO.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ I O */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_IO_Null.C b/src/bbcp_IO_Null.C index 52d3f4b..35964b2 100644 --- a/src/bbcp_IO_Null.C +++ b/src/bbcp_IO_Null.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ I O _ N u l l . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_IO_Null.h b/src/bbcp_IO_Null.h index 1d4c7ab..0a93d85 100644 --- a/src/bbcp_IO_Null.h +++ b/src/bbcp_IO_Null.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ I O _ N u l l */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_IO_Pipe.C b/src/bbcp_IO_Pipe.C index d7e21b2..a65c992 100644 --- a/src/bbcp_IO_Pipe.C +++ b/src/bbcp_IO_Pipe.C @@ -2,7 +2,7 @@ /* */ /* b b c p _ I O _ P i p e . C */ /* */ -/* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ +/*(c) 2011-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ /* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ diff --git a/src/bbcp_IO_Pipe.h b/src/bbcp_IO_Pipe.h index adb2d39..09148ff 100644 --- a/src/bbcp_IO_Pipe.h +++ b/src/bbcp_IO_Pipe.h @@ -4,8 +4,9 @@ /* */ /* b b c p _ I O _ P i p e */ /* */ -/* (c) 2011 by the Board of Trustees of the Leland Stanford, Jr., University */ -/* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2011-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ +/* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Link.C b/src/bbcp_Link.C index fe5c718..27c9134 100644 --- a/src/bbcp_Link.C +++ b/src/bbcp_Link.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ L i n k . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -47,7 +48,7 @@ extern bbcp_BuffPool bbcp_BPool; -extern bbcp_Config bbcp_Config; +extern bbcp_Config bbcp_Cfg; extern bbcp_Network bbcp_Net; @@ -65,8 +66,8 @@ int bbcp_Link::Wait = 0; bbcp_Link::bbcp_Link(int fd, const char *lname) : LinkNum(0), Buddy(0), Rendezvous(0), IOB(fd), Lname(strdup(lname)) { - csObj = (bbcp_Config.csOpts & bbcp_csLink - ? bbcp_ChkSum::Alloc(bbcp_Config.csType?bbcp_Config.csType:bbcp_csMD5) + csObj = (bbcp_Cfg.csOpts & bbcp_csLink + ? bbcp_ChkSum::Alloc(bbcp_Cfg.csType?bbcp_Cfg.csType:bbcp_csMD5) : 0); } @@ -84,7 +85,7 @@ int bbcp_Link::Buff2Net() // Establish logging options // - if (bbcp_Config.Options & bbcp_LOGOUT) IOB.Log(0, "NET"); + if (bbcp_Cfg.Options & bbcp_LOGOUT) IOB.Log(0, "NET"); // Do this until an error of eof // @@ -164,7 +165,7 @@ int bbcp_Link::Net2Buff() // Establish logging options // - if (bbcp_Config.Options & bbcp_LOGIN) IOB.Log("NET", 0); + if (bbcp_Cfg.Options & bbcp_LOGIN) IOB.Log("NET", 0); // Do this until an error of eof // @@ -213,7 +214,7 @@ int bbcp_Link::Net2Buff() // the buddy thread twice since that is the most that it may need to read. // - if (Nudge) {Wait = 0; i = bbcp_Config.Streams; + if (Nudge) {Wait = 0; i = bbcp_Cfg.Streams; do {Buddy->Rendezvous.Post();} while(i--); } if (notdone) @@ -254,9 +255,9 @@ int bbcp_Link::Control_In(bbcp_Buffer *bp) // if (hp->cmnd == (char)BBCP_CLCKS) {DEBUG("Checksum close request received on link " <bHdr.cksm); + bbcp_Cfg.setCS(bp->bHdr.cksm); bp->blen = 0; } bbcp_BPool.putFullBuff(bp); diff --git a/src/bbcp_Link.h b/src/bbcp_Link.h index 4d1427f..c688630 100644 --- a/src/bbcp_Link.h +++ b/src/bbcp_Link.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ L i n k . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_LogFile.C b/src/bbcp_LogFile.C index 1639bd8..4e91c82 100644 --- a/src/bbcp_LogFile.C +++ b/src/bbcp_LogFile.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ L o g F i l e . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_LogFile.h b/src/bbcp_LogFile.h index 3f1f273..14e2427 100644 --- a/src/bbcp_LogFile.h +++ b/src/bbcp_LogFile.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ L o g F i l e . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_MD5.h b/src/bbcp_MD5.h index 55e7ed2..d8476d8 100644 --- a/src/bbcp_MD5.h +++ b/src/bbcp_MD5.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ M D 5 . h */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_MD5_openssl.C b/src/bbcp_MD5_openssl.C index 3b88645..3b2a225 100644 --- a/src/bbcp_MD5_openssl.C +++ b/src/bbcp_MD5_openssl.C @@ -43,7 +43,11 @@ void bbcp_MD5_openssl::byteReverse(unsigned char *buf, unsigned longs) * initialization constants. */ void bbcp_MD5_openssl::MD5Init(MD5_CTX *ctx) { +#ifdef MACOS + CC_MD5_Init(ctx); +#else MD5_Init(ctx); +#endif } /* @@ -51,7 +55,11 @@ void bbcp_MD5_openssl::MD5Init(MD5_CTX *ctx) { * of bytes. */ void bbcp_MD5_openssl::MD5Update(MD5_CTX *ctx, unsigned char const *buf, unsigned len) { +#ifdef MACOS + CC_MD5_Update(ctx, buf, len); +#else MD5_Update(ctx, buf, len); +#endif } /* @@ -60,7 +68,11 @@ void bbcp_MD5_openssl::MD5Update(MD5_CTX *ctx, unsigned char const *buf, unsigne */ void bbcp_MD5_openssl::MD5Final(unsigned char digest[16], MD5_CTX *ctx) { +#ifdef MACOS + CC_MD5_Final(digest, ctx); +#else MD5_Final(digest, ctx); +#endif } #ifndef ASM_MD5 diff --git a/src/bbcp_MD5_openssl.h b/src/bbcp_MD5_openssl.h index 20804b5..6ec2715 100644 --- a/src/bbcp_MD5_openssl.h +++ b/src/bbcp_MD5_openssl.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ M D 5 . h */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -44,7 +45,13 @@ typedef unsigned int uint32; #include "bbcp_Headers.h" #include "bbcp_ChkSum.h" +#ifdef MACOS +#include "CommonCrypto/CommonDigest.h" +#undef MD5_CTX +#define MD5_CTX CC_MD5_CTX +#else #include "openssl/md5.h" +#endif class bbcp_MD5_openssl : public bbcp_ChkSum { diff --git a/src/bbcp_NetAddr.C b/src/bbcp_NetAddr.C index 965cb8d..976810b 100644 --- a/src/bbcp_NetAddr.C +++ b/src/bbcp_NetAddr.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ N e t A d d r . C */ /* */ -/*(c) 2013-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved */ +/*(c) 2013-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_NetAddr.h b/src/bbcp_NetAddr.h index d0ab541..4daf11d 100644 --- a/src/bbcp_NetAddr.h +++ b/src/bbcp_NetAddr.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ N e t A d d r . h */ /* */ -/*(c) 2013-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved */ +/*(c) 2013-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_NetAddrInfo.C b/src/bbcp_NetAddrInfo.C index d90b646..72209d9 100644 --- a/src/bbcp_NetAddrInfo.C +++ b/src/bbcp_NetAddrInfo.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ N e t A d d r I n f o . C */ /* */ -/*(c) 2013-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved */ +/*(c) 2013-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -173,6 +174,37 @@ int bbcp_NetAddrInfo::Format(char *bAddr, int bLen, fmtUse theFmt, int fmtOpts) return QFill(bAddr, bLen); return totLen+n; } + +/******************************************************************************/ +/* i s H o s t N a m e */ +/******************************************************************************/ + +bool bbcp_NetAddrInfo::isHostName(const char *name) +{ + const char *dot; + int dnum; + +// First check for Iv6 format or hostname +// + if (*name == '[') return false; + if (!isdigit(*name)) return true; + +// The IPv4 case is more complicated. The fastest way here is this is a +// host name if there are no dots or if the last component is not a digit +// according to the RFC spec. +// + if (!(dot = rindex(name, '.')) || !isdigit(*(dot+1))) return true; + +// We are not out of the woods yet. Now we need to do a full check. +// + name++; dnum = 0; + while(*name) + {if (*name == '.') dnum++; + else if (!isdigit(*name)) return true; + name++; + } + return (dnum == 3 ? false : true); +} /******************************************************************************/ /* i s L o o p b a c k */ diff --git a/src/bbcp_NetAddrInfo.h b/src/bbcp_NetAddrInfo.h index 20cdfd9..ad7dea0 100644 --- a/src/bbcp_NetAddrInfo.h +++ b/src/bbcp_NetAddrInfo.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ N e t A d d r I n f o . h */ /* */ -/*(c) 2013-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved */ +/*(c) 2013-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -93,6 +94,20 @@ static const int old6Map4 = 0x0000004; //!< Use deprecated IPV6 mapped format int Format(char *bAddr, int bLen, fmtUse fmtType=fmtAuto, int fmtOpts=0); +//------------------------------------------------------------------------------ +//! Indicate whether or not a string is a possible hostname and not IP address. +//! The return value does not aimply any kind of validity. For instance, a +//! false return indicates this is not a valid hostname does not mean it is +//! a valid IP address. +//! +//! @param name The string to check. +//! +//! @return True: This is a possible hostname (i.e. not IP address). +//! @return False: This is not a possible hostname. +//------------------------------------------------------------------------------ + +static bool isHostName(const char *name); + //------------------------------------------------------------------------------ //! Indicate whether or not our address is the loopback address. Use this //! method to gaurd against UDP packet spoofing. @@ -318,7 +333,7 @@ union {struct sockaddr *sockAddr; char *hostName; LocInfo addrLoc; socklen_t addrSize; +int sockNum; short protType; -short sockNum; }; #endif diff --git a/src/bbcp_NetLogger.C b/src/bbcp_NetLogger.C index 2249592..6cfdcf9 100644 --- a/src/bbcp_NetLogger.C +++ b/src/bbcp_NetLogger.C @@ -3,7 +3,8 @@ /* */ /* b b c p _ N e t L o g g e r . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_NetLogger.h b/src/bbcp_NetLogger.h index ff22479..92eb40f 100644 --- a/src/bbcp_NetLogger.h +++ b/src/bbcp_NetLogger.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ N e t L o g g e r . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_NetSockAddr.h b/src/bbcp_NetSockAddr.h index 1505149..011ec84 100644 --- a/src/bbcp_NetSockAddr.h +++ b/src/bbcp_NetSockAddr.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ N e t S o c k A d d r . h */ /* */ -/*(c) 2013-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved */ +/*(c) 2013-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Network.C b/src/bbcp_Network.C index 8ca3263..ec6394f 100644 --- a/src/bbcp_Network.C +++ b/src/bbcp_Network.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ N e t w o r k . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Network.h b/src/bbcp_Network.h index 1d8ad6b..3e01751 100644 --- a/src/bbcp_Network.h +++ b/src/bbcp_Network.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ N e t w o r k . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Node.C b/src/bbcp_Node.C index 24982bc..c723d41 100644 --- a/src/bbcp_Node.C +++ b/src/bbcp_Node.C @@ -3,7 +3,8 @@ /* b b c p _ N o d e . C */ /* */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -46,7 +47,7 @@ /* E x t e r n a l O b j e c t s */ /******************************************************************************/ -extern bbcp_Config bbcp_Config; +extern bbcp_Config bbcp_Cfg; extern bbcp_BuffPool bbcp_APool; extern bbcp_BuffPool bbcp_BPool; @@ -71,7 +72,7 @@ void *bbcp_doCX(void *pp) void *bbcp_doWrite(void *pp) { bbcp_File *fp = (bbcp_File *)pp; - long retc = fp->Write_All(bbcp_BPool, bbcp_Config.Streams); + long retc = fp->Write_All(bbcp_BPool, bbcp_Cfg.Streams); return (void *)retc; } void *bbcp_Buff2Net(void *link) @@ -92,7 +93,7 @@ void *bbcp_Connect(void *protp) bbcp_Link *link; int retc; - if ((link = bbcp_Net.Connect(bbcp_Config.CBhost, bbcp_Config.CBport))) + if ((link = bbcp_Net.Connect(bbcp_Cfg.CBhost, bbcp_Cfg.CBport))) {if ((retc = protocol->Login(link, 0)) < 0) {delete link; link = 0;} } @@ -132,7 +133,7 @@ int bbcp_Node::getBuffers(int isTrg, int isLZO) // Allocate the buffers // - return bbcp_BPool.Allocate(bbcp_Config.BNum, bbcp_Config.RWBsz, isTrg); + return bbcp_BPool.Allocate(bbcp_Cfg.BNum, bbcp_Cfg.RWBsz, isTrg); } /******************************************************************************/ @@ -164,7 +165,7 @@ int bbcp_Node::Put(char *data[], int dlen[]) cerr <= ArgvSize) return bbcp_Emsg("Run", -E2BIG, "starting", prog); - Argv[numa++] = parg; - Argv[numa] = 0; + Argv[numa] = 0; // Invoke the program // - if ((fderr=NStream.Exec(Argv, 1, bbcp_Config.MLog != 0)) < 0) return -fderr; + if ((fderr=NStream.Exec(Argv, 1, bbcp_Cfg.MLog != 0)) < 0) return -fderr; // Perform logging function here // - if (bbcp_Config.MLog) bbcp_Config.MLog->Monitor(fderr, parg); + if (bbcp_Cfg.MLog) bbcp_Cfg.MLog->Monitor(fderr, parg); // Perform debugging here // @@ -264,7 +274,7 @@ void bbcp_Node::Stop(int Report) // If window reporting wanted do so only if very verbose and autotuning // - if (bbcp_Config.Options & bbcp_BLAB && bbcp_Net.AutoTune() && data_link[0]) + if (bbcp_Cfg.Options & bbcp_BLAB && bbcp_Net.AutoTune() && data_link[0]) chkWsz(data_link[0]->FD(), 1); // Kill any attached process @@ -320,6 +330,7 @@ int bbcp_Node::RecvFile(bbcp_FileSpec *fp, bbcp_Node *Remote) static const int wOnly = S_IWUSR; const char *Args = 0, *Act = "opening", *Path = fp->targpath; + char *tmpPath = 0; long tretc = 0; int i, oflag, retc, Mode = wOnly, progtid = 0; long long startoff = 0; @@ -333,11 +344,11 @@ int bbcp_Node::RecvFile(bbcp_FileSpec *fp, bbcp_Node *Remote) // Perform Force or Append processing // - if (bbcp_Config.Options & bbcp_XPIPE) + if (bbcp_Cfg.Options & bbcp_XPIPE) {oflag = O_WRONLY; - Path = bbcp_Config.snkSpec->pathname; - Args = bbcp_Config.snkSpec->fileargs; - if (bbcp_Config.snkSpec->Info.Otype != 'p') Act = "running"; + Path = bbcp_Cfg.snkSpec->pathname; + Args = bbcp_Cfg.snkSpec->fileargs; + if (bbcp_Cfg.snkSpec->Info.Otype != 'p') Act = "running"; else {Mode |= S_IFIFO; if (Args) {bbcp_Fmsg("RecvFile", @@ -346,12 +357,12 @@ int bbcp_Node::RecvFile(bbcp_FileSpec *fp, bbcp_Node *Remote) } } } - else if (bbcp_Config.Options & bbcp_FORCE) - {if (!(bbcp_Config.Options & bbcp_NOUNLINK)) + else if (bbcp_Cfg.Options & bbcp_FORCE) + {if (!(bbcp_Cfg.Options & bbcp_NOUNLINK)) fp->FSys()->RM(Path); oflag = O_WRONLY | O_CREAT | O_TRUNC; } - else if (bbcp_Config.Options & bbcp_APPEND) + else if (bbcp_Cfg.Options & bbcp_APPEND) {if ((retc = fp->WriteSigFile())) return retc; if ((startoff = fp->targetsz)) oflag = O_WRONLY; else oflag = O_CREAT | O_WRONLY; @@ -360,14 +371,14 @@ int bbcp_Node::RecvFile(bbcp_FileSpec *fp, bbcp_Node *Remote) // Establish mode, we normally make the file write-only // - if ( bbcp_Config.Options & bbcp_RTCSNK - && !(bbcp_Config.Options & (bbcp_RTCHIDE|bbcp_XPIPE))) - Mode = bbcp_Config.Mode|S_IWUSR|S_ISUID; + if ( bbcp_Cfg.Options & bbcp_RTCSNK + && !(bbcp_Cfg.Options & (bbcp_RTCHIDE|bbcp_XPIPE))) + Mode = bbcp_Cfg.Mode|S_IWUSR|S_ISUID; // Tell the user what we are bout to do // - if ((bbcp_Config.Options & bbcp_BLAB) | bbcp_Config.Progint) - if (bbcp_Config.Options & bbcp_APPEND) + if ((bbcp_Cfg.Options & bbcp_BLAB) | bbcp_Cfg.Progint) + if (bbcp_Cfg.Options & bbcp_APPEND) {char buff[32]; sprintf(buff, "%lld", startoff); bbcp_Fmsg("RecvFile","Appending to",Path,"at offset",buff); @@ -385,7 +396,7 @@ int bbcp_Node::RecvFile(bbcp_FileSpec *fp, bbcp_Node *Remote) DEBUG("Waiting for child " <FSys()->DirectIO(1); + if (bbcp_Cfg.Options & bbcp_ODIO) {fp->FSys()->DirectIO(1); DEBUG("Direct output requested.");} +// Make temporary file name if specified +// + if (bbcp_Cfg.Options & bbcp_RTTMP) + {tmpPath = new char[strlen(Path)+1+4]; + strcpy(tmpPath,Path); + strcat(tmpPath,"_tmp"); + Path = tmpPath; + } + // Open the file and set the starting offset // Elapsed_Timer.Start(); @@ -414,7 +434,7 @@ int bbcp_Node::RecvFile(bbcp_FileSpec *fp, bbcp_Node *Remote) // If compression is wanted, set up the compression objects // - if (bbcp_Config.Options & bbcp_COMPRESS + if (bbcp_Cfg.Options & bbcp_COMPRESS && !(cxp = setup_CX(0, outFile->ioFD()))) return -ECANCELED; // Start a thread for each data link we have @@ -431,7 +451,7 @@ int bbcp_Node::RecvFile(bbcp_FileSpec *fp, bbcp_Node *Remote) // If we are compressing start the sequence thread now // - if (bbcp_Config.Options & bbcp_COMPRESS) + if (bbcp_Cfg.Options & bbcp_COMPRESS) {seqFile = new bbcp_File(Path, 0, 0); if ((retc = bbcp_Thread_Start(bbcp_doWrite, (void *)seqFile, &tid))<0) {bbcp_Emsg("RecvFile",retc,"starting disk thread for",Path); @@ -447,21 +467,21 @@ int bbcp_Node::RecvFile(bbcp_FileSpec *fp, bbcp_Node *Remote) // If a periodic progress message is wanted, start a progress thread // - if (bbcp_Config.Progint) + if (bbcp_Cfg.Progint) {pmp = new bbcp_ProgMon(); - pmp->Start(outFile, cxp, bbcp_Config.Progint, fp->Info.size-startoff); + pmp->Start(outFile, cxp, bbcp_Cfg.Progint, fp->Info.size-startoff); } // Write the whole file // - if (bbcp_Config.Options & bbcp_COMPRESS) + if (bbcp_Cfg.Options & bbcp_COMPRESS) retc = outFile->Write_All(bbcp_APool, 1); - else retc = outFile->Write_All(bbcp_BPool, bbcp_Config.Streams); + else retc = outFile->Write_All(bbcp_BPool, bbcp_Cfg.Streams); DEBUG("File write ended; rc=" <TID))) retc = 128; DEBUG("File expansion ended; rc=" <Info.size && Info.mode - && !(bbcp_Config.Options & bbcp_NOFSZCHK)) + && !(bbcp_Cfg.Options & bbcp_NOFSZCHK)) {const char *What = (Info.size < fp->Info.size ? "Not all" : "Too much"); retc = 29; @@ -498,10 +518,19 @@ int bbcp_Node::RecvFile(bbcp_FileSpec *fp, bbcp_Node *Remote) } } DEBUG("Outfile " <targpath) !=0 ) + bbcp_Emsg("RecvFile", errno, "rename", Path); + delete[] tmpPath; + Path=fp->targpath; + } + // Report detailed I/O stats, if so wanted // Elapsed_Timer.Stop(); - if (!retc && bbcp_Config.Options & bbcp_VERBOSE) + if (!retc && bbcp_Cfg.Options & bbcp_VERBOSE) {double ttime; Elapsed_Timer.Report(ttime); Report(ttime, fp, outFile, cxp); @@ -536,7 +565,7 @@ int bbcp_Node::SendFile(bbcp_FileSpec *fp) // Set open options (check for pipes) // - if (bbcp_Config.Options & bbcp_XPIPE) + if (bbcp_Cfg.Options & bbcp_XPIPE) {if (fp->Info.Otype == 'p') Mode = S_IFIFO; else Act = "running"; } @@ -551,7 +580,7 @@ int bbcp_Node::SendFile(bbcp_FileSpec *fp) Parent_Monitor.Start(); retc = bbcp_OS.Waitpid(Child); Parent_Monitor.Stop(); - if (bbcp_Config.Options & bbcp_BLAB) + if (bbcp_Cfg.Options & bbcp_BLAB) write(STDERR_FILENO, buff, Usage("Source", buff, sizeof(buff))); return retc; } @@ -562,11 +591,11 @@ int bbcp_Node::SendFile(bbcp_FileSpec *fp) // Set Concurrency // - bbcp_Thread_MT(bbcp_Config.MTLevel); + bbcp_Thread_MT(bbcp_Cfg.MTLevel); // Request direct I/O if so wanted // - if (bbcp_Config.Options & bbcp_IDIO) {fp->FSys()->DirectIO(1); + if (bbcp_Cfg.Options & bbcp_IDIO) {fp->FSys()->DirectIO(1); DEBUG("Direct input requested.");} // Open the input file and set starting offset @@ -580,7 +609,7 @@ int bbcp_Node::SendFile(bbcp_FileSpec *fp) // If compression is wanted, set up the compression objects // - if (bbcp_Config.Options & bbcp_COMPRESS + if (bbcp_Cfg.Options & bbcp_COMPRESS && !(cxp = setup_CX(1, inFile->ioFD()))) return -ECANCELED; // Start a thread for each data link we have @@ -602,20 +631,20 @@ int bbcp_Node::SendFile(bbcp_FileSpec *fp) // Start the Transfer Time Limit // - if (bbcp_Config.TimeLimit) + if (bbcp_Cfg.TimeLimit) {TLimit = new bbcp_ProcMon(); - TLimit->Start(bbcp_Config.TimeLimit, &bbcp_BPool); + TLimit->Start(bbcp_Cfg.TimeLimit, &bbcp_BPool); } // Read the whole file // - if (bbcp_Config.Options & bbcp_COMPRESS) retc=inFile->Read_All(bbcp_APool,1); - else retc = inFile->Read_All(bbcp_BPool, bbcp_Config.Bfact); + if (bbcp_Cfg.Options & bbcp_COMPRESS) retc=inFile->Read_All(bbcp_APool,1); + else retc = inFile->Read_All(bbcp_BPool, bbcp_Cfg.Bfact); DEBUG("File read ended; rc=" <TID))) retc = 128; DEBUG("File compression ended; rc=" <Login(link, 1))) {link->LinkNum = dlcount; data_link[dlcount++] = link;} @@ -704,12 +733,12 @@ int bbcp_Node::Incomming(bbcp_Protocol *protocol) // Unbind the network and make sure we have all of the agreed upon links // bbcp_Net.unBind(); - if (dlcount < bbcp_Config.Streams) return Recover("Accept"); + if (dlcount < bbcp_Cfg.Streams) return Recover("Accept"); iocount = dlcount; // Initialize the buddy pipeline; a patented way of ensuring maximum parallelism // - if (dlcount > 1 && (bbcp_Config.Options & (bbcp_SRC|bbcp_ORDER))) + if (dlcount > 1 && (bbcp_Cfg.Options & (bbcp_SRC|bbcp_ORDER))) {i = dlcount-1; data_link[i]->setBuddy(data_link[0]); while(i--) data_link[i]->setBuddy(data_link[i+1]); @@ -718,7 +747,7 @@ int bbcp_Node::Incomming(bbcp_Protocol *protocol) // Determine what the actual window size is (only if verbose) // - if (bbcp_Config.Options & bbcp_BLAB) chkWsz(data_link[0]->FD()); + if (bbcp_Cfg.Options & bbcp_BLAB) chkWsz(data_link[0]->FD()); return 0; } @@ -733,7 +762,7 @@ int bbcp_Node::Outgoing(bbcp_Protocol *protocol) // Establish the control connection first // - if ((link = bbcp_Net.Connect(bbcp_Config.CBhost, bbcp_Config.CBport, 3))) + if ((link = bbcp_Net.Connect(bbcp_Cfg.CBhost, bbcp_Cfg.CBport, 3))) if ((retc = protocol->Login(link, 0)) < 0) {delete link; link = 0;} @@ -744,7 +773,7 @@ int bbcp_Node::Outgoing(bbcp_Protocol *protocol) // Start threads for data connections // - for (i = 0; i < bbcp_Config.Streams; i++) + for (i = 0; i < bbcp_Cfg.Streams; i++) {if ((retc=bbcp_Thread_Start(bbcp_Connect,(void *)protocol,&tid))<0) {bbcp_Emsg("Outgoing", retc, "starting connect thread"); _exit(100); @@ -754,7 +783,7 @@ int bbcp_Node::Outgoing(bbcp_Protocol *protocol) usleep(1); #endif } - for (i = 0; i < bbcp_Config.Streams; i++) + for (i = 0; i < bbcp_Cfg.Streams; i++) {if (!(link = (bbcp_Link *)bbcp_Thread_Wait(T_id[i]))) break; link->LinkNum = dlcount; data_link[dlcount++] = link; } @@ -763,15 +792,15 @@ int bbcp_Node::Outgoing(bbcp_Protocol *protocol) // Make sure we have all of the required links // - if (dlcount < bbcp_Config.Streams) return Recover("Connect"); + if (dlcount < bbcp_Cfg.Streams) return Recover("Connect"); // Determine what the actual window size is (only if verbose) // - if (bbcp_Config.Options & bbcp_BLAB) chkWsz(data_link[0]->FD()); + if (bbcp_Cfg.Options & bbcp_BLAB) chkWsz(data_link[0]->FD()); // Initialize the buddy pipeline; a patented way of ensuring maximum parallelism // - if (dlcount > 1 && (bbcp_Config.Options & (bbcp_SRC|bbcp_ORDER))) + if (dlcount > 1 && (bbcp_Cfg.Options & (bbcp_SRC|bbcp_ORDER))) {i = dlcount-1; data_link[i]->setBuddy(data_link[0]); while(i--) data_link[i]->setBuddy(data_link[i+1]); @@ -787,7 +816,7 @@ int bbcp_Node::Outgoing(bbcp_Protocol *protocol) int bbcp_Node::Recover(const char *who) { char mbuff[256]; - sprintf(mbuff, "%d of %d data streams.", dlcount, bbcp_Config.Streams); + sprintf(mbuff, "%d of %d data streams.", dlcount, bbcp_Cfg.Streams); bbcp_Fmsg(who, "Unable to allocate more than", mbuff); while(dlcount) data_link[--dlcount]->Close(); return -1; @@ -822,7 +851,7 @@ int n; n = sprintf(Line, "File %s created; %lld bytes at %.1f %sB/s%s\n", fp->targpath, xbytes, xrate, xType, buff); write(STDERR_FILENO, Line, n); - if (!(bbcp_Config.Options & bbcp_BLAB)) return; + if (!(bbcp_Cfg.Options & bbcp_BLAB)) return; // Tell user how many reorder events there were // @@ -848,7 +877,7 @@ bbcp_ZCX *bbcp_Node::setup_CX(int deflating, int iofd) {ibp = &bbcp_APool; rbp = &bbcp_APool; obp = &bbcp_BPool; - clvl = (bbcp_Config.Complvl ? bbcp_Config.Complvl : 1); + clvl = (bbcp_Cfg.Complvl ? bbcp_Cfg.Complvl : 1); } else { ibp = &bbcp_CPool; rbp = &bbcp_BPool; @@ -858,13 +887,13 @@ bbcp_ZCX *bbcp_Node::setup_CX(int deflating, int iofd) // Allocate buffers in the A pool // - if (bbcp_APool.Allocate(bbcp_Config.BNum, bbcp_Config.RWBsz, !deflating)) + if (bbcp_APool.Allocate(bbcp_Cfg.BNum, bbcp_Cfg.RWBsz, !deflating)) return 0; // Allocate a new compression/expansion object // cxp = new bbcp_ZCX(ibp, rbp, obp, clvl, iofd, - (int)(bbcp_Config.Options & (clvl ? bbcp_LOGCMP : bbcp_LOGEXP))); + (int)(bbcp_Cfg.Options & (clvl ? bbcp_LOGCMP : bbcp_LOGEXP))); // Start the compression/expansion thread // diff --git a/src/bbcp_Node.h b/src/bbcp_Node.h index 0ba4b7d..82936ff 100644 --- a/src/bbcp_Node.h +++ b/src/bbcp_Node.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ N o d e . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -29,6 +30,7 @@ #include #include +#include "bbcp_Config.h" #include "bbcp_FileSpec.h" #include "bbcp_File.h" #include "bbcp_Link.h" diff --git a/src/bbcp_Platform.h b/src/bbcp_Platform.h index 94ac855..72bf8a5 100644 --- a/src/bbcp_Platform.h +++ b/src/bbcp_Platform.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ P l a t f o r m . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_ProcMon.C b/src/bbcp_ProcMon.C index d8d0b2e..e86b34b 100644 --- a/src/bbcp_ProcMon.C +++ b/src/bbcp_ProcMon.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ P r o c M o n . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -29,7 +30,6 @@ #include #include #include -#include "bbcp_Config.h" #include "bbcp_Debug.h" #include "bbcp_Emsg.h" #include "bbcp_Headers.h" diff --git a/src/bbcp_ProcMon.h b/src/bbcp_ProcMon.h index 41687d1..d5942e7 100644 --- a/src/bbcp_ProcMon.h +++ b/src/bbcp_ProcMon.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ P r o c M o n . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_ProgMon.C b/src/bbcp_ProgMon.C index 0c76e87..d2dad06 100644 --- a/src/bbcp_ProgMon.C +++ b/src/bbcp_ProgMon.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ P r o g M o n . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -40,7 +41,7 @@ /* G l o b a l V a r i a b l e s */ /******************************************************************************/ -extern bbcp_Config bbcp_Config; +extern bbcp_Config bbcp_Cfg; /******************************************************************************/ /* E x t e r n a l I n t e r f a c e R o u t i n e s */ @@ -69,7 +70,7 @@ void bbcp_ProgMon::Monitor() bbcp_Timer etime; char buff[200], pbuff[24], tbuff[24], cxinfo[40], *cxip; const char *xtXB, *xaXB; - int bewordy = bbcp_Config.Options & bbcp_VERBOSE; + int bewordy = bbcp_Cfg.Options & bbcp_VERBOSE; // Determine whether we need to report compression ratio (assume no percentages) // @@ -104,7 +105,7 @@ void bbcp_ProgMon::Monitor() lasttime = elptime; xtXB = bbcp_Config::Scale(xfrtime); - if (bbcp_Config.Logfn) *tbuff = 0; + if (bbcp_Cfg.Logfn) *tbuff = 0; else etime.Format(tbuff); if (bewordy) sprintf(buff, "bbcp: %s %s done; %.1f %sB/s, " diff --git a/src/bbcp_ProgMon.h b/src/bbcp_ProgMon.h index 24a6d7f..c9b7c1c 100644 --- a/src/bbcp_ProgMon.h +++ b/src/bbcp_ProgMon.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ P r o g M o n . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Protocol.C b/src/bbcp_Protocol.C index 375dbb5..ab31c05 100644 --- a/src/bbcp_Protocol.C +++ b/src/bbcp_Protocol.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ P r o t o c o l . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -28,11 +29,14 @@ #include #include #include +#include #include #include +#include #include "bbcp_Config.h" #include "bbcp_Emsg.h" #include "bbcp_Headers.h" +#include "bbcp_NetAddr.h" #include "bbcp_Network.h" #include "bbcp_Node.h" #include "bbcp_Protocol.h" @@ -55,11 +59,33 @@ bbcp_Node *np; {if (np) delete np;} }; +/******************************************************************************/ +/* L o c a l O b j e c t s */ +/******************************************************************************/ + +namespace +{ + +struct Cmp_Key +{ + + bool operator()(char const *a, char const *b) const + { +//cerr <<"Comp " < CopySet; + +} + /******************************************************************************/ /* E x t e r n a l O b j e c t s */ /******************************************************************************/ -extern bbcp_Config bbcp_Config; +extern bbcp_Config bbcp_Cfg; extern bbcp_Network bbcp_Net; @@ -79,8 +105,8 @@ int bbcp_Protocol::Schedule(bbcp_Node *Fnode, bbcp_FileSpec *Ffs, { int retc; - char *cbhost, *addOpt[2]; - bool fcbh = false; + char *cbhost, *addOpt[2], *hDest; + bool fcbh = false, noDNS = (bbcp_Cfg.Options & bbcp_NODNS) != 0; // Start-up the first node // @@ -90,11 +116,11 @@ int bbcp_Protocol::Schedule(bbcp_Node *Fnode, bbcp_FileSpec *Ffs, // Determine additional options // if (Ftype[1] == 'R') - {addOpt[0] = bbcp_Config.CopyOSrc; - addOpt[1] = bbcp_Config.CopyOTrg; + {addOpt[0] = bbcp_Cfg.CopyOSrc; + addOpt[1] = bbcp_Cfg.CopyOTrg; } else { - addOpt[0] = bbcp_Config.CopyOTrg; - addOpt[1] = bbcp_Config.CopyOSrc; + addOpt[0] = bbcp_Cfg.CopyOTrg; + addOpt[1] = bbcp_Cfg.CopyOSrc; } // Send the arguments @@ -103,7 +129,7 @@ int bbcp_Protocol::Schedule(bbcp_Node *Fnode, bbcp_FileSpec *Ffs, // Get the callback port from the first host // - if ((retc = getCBPort(Fnode))) return retc; + if ((retc = getCBPort(Fnode, hDest))) return retc; // Start the second node // @@ -112,14 +138,36 @@ int bbcp_Protocol::Schedule(bbcp_Node *Fnode, bbcp_FileSpec *Ffs, // Compute callback hostname and reset callback port // - if (!(Ffs->hostname)) cbhost = bbcp_Config.MyAddr; - else if ((bbcp_Config.Options & bbcp_NODNS && isdigit(Ffs->hostname[0])) - || Ffs->hostname[0] == '[') cbhost = Ffs->hostname; - else fcbh = (cbhost = bbcp_Net.FullHostName(Ffs->hostname,1)) !=0; + if (hDest && (noDNS || !(bbcp_Cfg.Options & bbcp_VERBOSE))) + {cbhost = hDest; hDest = 0; fcbh = true;} + else if (!(Ffs->hostname)) cbhost = bbcp_Cfg.MyAddr; + else if (noDNS && !bbcp_NetAddrInfo::isHostName(Ffs->hostname)) + cbhost = Ffs->hostname; + else fcbh = (cbhost = bbcp_Net.FullHostName(Ffs->hostname,(noDNS?1:0))) !=0; + +// If the node send us it's actual host name use it preferentially and notify +// the user if we switched hosts if we need to do that. +// + if (hDest) + {if (cbhost) + {if (!noDNS && (bbcp_Cfg.Options & bbcp_VERBOSE) + && strcmp(cbhost, hDest)) Chk4Redir(cbhost, hDest); + if (fcbh) free(cbhost); + } + cbhost = hDest; + fcbh = true; + } + +// After all of this, verify we really do have a callback host +// + if (!cbhost) + {bbcp_Fmsg("Protocol", "Unable to determine the callback host!"); + return EADDRNOTAVAIL; + } // Send the arguments // - retc = SendArgs(Lnode, Lfs, cbhost, bbcp_Config.CBport, addOpt[1]); + retc = SendArgs(Lnode, Lfs, cbhost, bbcp_Cfg.CBport, addOpt[1]); if (fcbh) free(cbhost); if (retc) return retc; @@ -143,19 +191,25 @@ int bbcp_Protocol::Schedule(bbcp_Node *Fnode, bbcp_FileSpec *Ffs, /* g e t C B P o r t */ /******************************************************************************/ -int bbcp_Protocol::getCBPort(bbcp_Node *Node) +int bbcp_Protocol::getCBPort(bbcp_Node *Node, char *&hDest) { char *wp; int pnum; -// The remote program should hve started, get the call back port +// The remote program should hve started, get the call back port. New versions +// will also supply their host name which we always use if present. // if ((wp = Node->GetLine())) {if ((wp = Node->GetToken()) && !strcmp(wp, "200") && (wp = Node->GetToken()) && !strcmp(wp, "Port:") && (wp = Node->GetToken()) - && bbcp_Config.a2n("callback port", wp, pnum, 0, 65535) == 0) - {bbcp_Config.CBport = pnum; return 0;} + && bbcp_Cfg.a2n("callback port", wp, pnum, 0, 65535) == 0) + {bbcp_Cfg.CBport = pnum; + if ((wp = Node->GetToken()) && !strcmp(wp, "Host:") + && (wp = Node->GetToken())) hDest = strdup(wp); + else hDest = 0; + return 0; + } } // Invalid response @@ -171,9 +225,9 @@ int bbcp_Protocol::getCBPort(bbcp_Node *Node) int bbcp_Protocol::setCBPort(int pnum) { -// The port number simply gets sent via standard out +// The port number and our host name simply get sent via standard out // - cout <<"200 Port: " <Stat(); - if (fp->Info.Otype == 'd' && !(bbcp_Config.Options & bbcp_RECURSE) + if (fp->Info.Otype == 'd' && !(bbcp_Cfg.Options & bbcp_RECURSE) && fp->Info.size) {bbcp_Fmsg("Source", fp->pathname, "is a directory."); NoGo = 1; break; @@ -253,7 +307,7 @@ int bbcp_Protocol::Process(bbcp_Node *Node) // If this is a recursive list, do it in the bacground while we try to connect. // This avoids time-outs when large number of files are enumerated. // - if (!NoGo && bbcp_Config.Options & bbcp_RECURSE) + if (!NoGo && bbcp_Cfg.Options & bbcp_RECURSE) if ((rc = bbcp_Thread_Start(bbcp_FileSpecIndex, 0, &Tid)) < 0) {bbcp_Emsg("Protocol", rc, "starting file enumeration thread."); NoGo = 1; @@ -261,15 +315,15 @@ int bbcp_Protocol::Process(bbcp_Node *Node) // Establish all connections // - if (Node->Start(this, (bbcp_Config.Options & bbcp_CON2SRC)) + if (Node->Start(this, (bbcp_Cfg.Options & bbcp_CON2SRC)) || Node->getBuffers(0)) return 2; Local = Node; // At this point, if we're running with the -r recursive option, our list of -// file specs (bbcp_Config.srcSpec) is being extended recursively to include +// file specs (bbcp_Cfg.srcSpec) is being extended recursively to include // all subdirs and their contents. We must wait for the thread to finish. // - if (!NoGo && bbcp_Config.Options & bbcp_RECURSE) + if (!NoGo && bbcp_Cfg.Options & bbcp_RECURSE) NoGo = (bbcp_Thread_Wait(Tid) != 0); // If there was a fatal error, we can exit now, the remote side will exit @@ -320,7 +374,7 @@ int bbcp_Protocol::Process_exit() // if (!(wp = Remote->GetToken())) {bbcp_Fmsg("Process_exit", "missing return code."); retc = 22;} - else if (bbcp_Config.a2n("return code", wp, retc, 0, 255)) retc = 22; + else if (bbcp_Cfg.a2n("return code", wp, retc, 0, 255)) retc = 22; // Return to caller // @@ -333,8 +387,8 @@ int bbcp_Protocol::Process_exit() int bbcp_Protocol::Process_flist() { - bbcp_FileSpec *dp = bbcp_Config.srcPath; - bbcp_FileSpec *fp = bbcp_Config.srcSpec; + bbcp_FileSpec *dp = bbcp_Cfg.srcPath; + bbcp_FileSpec *fp = bbcp_Cfg.srcSpec; char buff[4096]; int blen; const char eoltag[] = "eol\n"; @@ -343,15 +397,18 @@ int bbcp_Protocol::Process_flist() // Simply go through the list of paths and report them back to the caller while(dp) - {if ((blen = dp->Encode(buff,(size_t)sizeof(buff))) < 0) return -1; - if (Remote->Put(buff, blen)) return -1; + {if (!(dp->isEmpty)) + {if ((blen = dp->Encode(buff,(size_t)sizeof(buff))) < 0) return -1; + if (Remote->Put(buff, blen)) return -1; + } dp = dp->next; } // Simply go through the list of files and report them back to the caller while(fp) - {if (fp->Info.Otype != 'd' || *(fp->filename)) + {if ((fp->Info.Otype == 'd' && !(fp->isEmpty)) + || (fp->Info.Otype != 'd' && *(fp->filename))) {if ((blen = fp->Encode(buff,(size_t)sizeof(buff))) < 0) return -1; if (Remote->Put(buff, blen)) return -1; } @@ -369,7 +426,7 @@ int bbcp_Protocol::Process_flist() int bbcp_Protocol::Process_get() { - bbcp_FileSpec *pp = 0, *fp = bbcp_Config.srcSpec; + bbcp_FileSpec *pp = 0, *fp = bbcp_Cfg.srcSpec; char *wp; int retc; int fnum, snum; @@ -381,10 +438,10 @@ int bbcp_Protocol::Process_get() // if (!(wp = Remote->GetToken())) {bbcp_Fmsg("Process_get", "missing stream number."); return 19;} - if (bbcp_Config.a2n("stream number", wp, snum, 0, 255)) return 19; + if (bbcp_Cfg.a2n("stream number", wp, snum, 0, 255)) return 19; if (!(wp = Remote->GetToken())) {bbcp_Fmsg("Process_get", "missing file number."); return 19;} - if (bbcp_Config.a2n("file number", wp, fnum, 0, 0x7ffffff)) return 19; + if (bbcp_Cfg.a2n("file number", wp, fnum, 0, 0x7ffffff)) return 19; if (!(wp = Remote->GetToken())) {bbcp_Fmsg("Process_get", "missing file name."); return 19;} @@ -402,13 +459,13 @@ int bbcp_Protocol::Process_get() // Unchain the file specification (get allowed only once) // if (pp) pp->next = fp->next; - else bbcp_Config.srcSpec = fp->next; + else bbcp_Cfg.srcSpec = fp->next; fp->next = 0; // Get the optional offset // if ((wp = Remote->GetToken())) - {if (bbcp_Config.a2ll("file offset", wp, foffset, 0, -1)) return 22; + {if (bbcp_Cfg.a2ll("file offset", wp, foffset, 0, -1)) return 22; if (foffset > fp->Info.size) {char buff[128]; sprintf(buff, "(%lld>%lld)", foffset, fp->Info.size); @@ -460,7 +517,7 @@ int bbcp_Protocol::Process_login(bbcp_Link *Net) // if (!(wp = np->GetToken()) || strcmp(wp, "login") || !(wp = np->GetToken()) || strcmp(wp, id) - || !(wp = np->GetToken()) || strcmp(wp, bbcp_Config.SecToken)) + || !(wp = np->GetToken()) || strcmp(wp, bbcp_Cfg.SecToken)) return bbcp_Fmsg("Process_Login", "Invalid login from", Net->LinkName()); // We are all done if this is not a control stream @@ -492,12 +549,12 @@ int bbcp_Protocol::Process_login(bbcp_Link *Net) // We can now do a window/buffer adjustment // - if (!wp) respWS = bbcp_Config.Wsize; + if (!wp) respWS = bbcp_Cfg.Wsize; else if (!(respWS = AdjustWS(wp, bp, 0))) return -1; // Respond to this login request (control only gets a response) // - blen = sprintf(buff, "204 loginok wsz: %d %d\n",respWS,bbcp_Config.RWBsz); + blen = sprintf(buff, "204 loginok wsz: %d %d\n",respWS,bbcp_Cfg.RWBsz); if ((retc = np->Put(buff, blen)) < 0) return -1; // All done @@ -514,56 +571,67 @@ int bbcp_Protocol::Process_login(bbcp_Link *Net) int bbcp_Protocol::Request(bbcp_Node *Node) { long long totsz=0; - bbcp_FileSpec *fp; + bbcp_FileSpec *dp, *fp; const char *dRM = 0; char buff[1024]; int retc, numfiles, numlinks, texists; - int outDir = (bbcp_Config.Options & bbcp_OUTDIR) != 0; - bool dotrim = false; + int outDir = (bbcp_Cfg.Options & bbcp_OUTDIR) != 0; + bool dotrim = false; + bool doPcopy = (bbcp_Cfg.Options & bbcp_PCOPY) != 0; + bool setDMode= (bbcp_Cfg.ModeD != bbcp_Cfg.ModeDC) != 0; + bool isAppend= (bbcp_Cfg.Options & bbcp_APPEND) != 0; // Establish all connections // - if (Node->Start(this, !(bbcp_Config.Options & bbcp_CON2SRC)) + if (Node->Start(this, !(bbcp_Cfg.Options & bbcp_CON2SRC)) || Node->getBuffers(1)) return 2; Local = Node; // Determine if the target exists // - texists = !bbcp_Config.snkSpec->Stat(0); - fs_obj = bbcp_Config.snkSpec->FSys(); + texists = !bbcp_Cfg.snkSpec->Stat(0); + fs_obj = bbcp_Cfg.snkSpec->FSys(); + +// Make sure we have a filesystem here +// + if (!fs_obj) + {bbcp_Fmsg("Request","Target directory", bbcp_Cfg.snkSpec->pathname, + "is not in a known file system"); + return Request_exit(2, 0); + } // If the target does not exist and we are doing a recursive copy, then we // presume that the target should be a directory and we should create it. // - if (!texists && (outDir || (bbcp_Config.Options & bbcp_RECURSE)) - && (bbcp_Config.Options & bbcp_AUTOMKD)) - {retc = fs_obj->MKDir(bbcp_Config.snkSpec->pathname, bbcp_Config.ModeDC); + if (!texists && (outDir || (bbcp_Cfg.Options & bbcp_RECURSE)) + && (bbcp_Cfg.Options & bbcp_AUTOMKD)) + {retc = fs_obj->MKDir(bbcp_Cfg.snkSpec->pathname, bbcp_Cfg.ModeDC); if (retc) Request_exit(retc); - texists = !bbcp_Config.snkSpec->Stat(0); - dotrim = !outDir; dRM = bbcp_Config.snkSpec->pathname; + texists = !bbcp_Cfg.snkSpec->Stat(0); + dotrim = !outDir; dRM = bbcp_Cfg.snkSpec->pathname; +//cerr <<"dotrim=" <pathname,"not found"); + bbcp_Cfg.snkSpec->pathname,"not found"); return Request_exit(2, dRM); } tdir_id = Tinfo.fileid; @@ -573,7 +641,7 @@ int bbcp_Protocol::Request(bbcp_Node *Node) // numfiles = Request_flist(totsz, numlinks, dotrim); if (numfiles < 0) return Request_exit(22, dRM); - if (numfiles == 0 && numlinks == 0 && !(bbcp_Config.Options & bbcp_GROSS)) + if (numfiles == 0 && numlinks == 0 && !(bbcp_Cfg.Options & bbcp_AUTOMKD)) {cout <<"200 End: 0 0" < 1 || numlinks > 1 || outDir) {if (!texists) {bbcp_Fmsg("Request", "Target directory", - bbcp_Config.snkSpec->pathname, "not found."); + bbcp_Cfg.snkSpec->pathname, "not found."); return Request_exit(2); } - if (bbcp_Config.snkSpec->Info.Otype != 'd') - {bbcp_Fmsg("Request", "Target", bbcp_Config.snkSpec->pathname, + if (bbcp_Cfg.snkSpec->Info.Otype != 'd') + {bbcp_Fmsg("Request", "Target", bbcp_Cfg.snkSpec->pathname, "is not a directory."); return Request_exit(20); } - bbcp_Config.Options |= bbcp_OUTDIR; + bbcp_Cfg.Options |= bbcp_OUTDIR; } // Make sure we have enough space in the filesystem // DEBUG("Preparing to copy " <NodeName()); return Request_exit(28, dRM); } -// Create all of the required directories +// Get each source file that isn't inside a directory (non-recursive copy) // - retc = 0; - fp = bbcp_Config.srcPath; - while(fp && !(retc = fp->Create_Path())) fp = fp->next; - if (retc) return Request_exit(retc); - -// Create all of the required symlinks -// - retc = 0; - fp = bbcp_Config.slkPath; - while(fp && !(retc = fp->Create_Link())) fp = fp->next; - if (retc) return Request_exit(retc); - -// Get each source file -// - fp = bbcp_Config.srcSpec; + fp = bbcp_Cfg.srcSpec; while(fp && !(retc=Request_get(fp)) && !(retc=Local->RecvFile(fp,Remote))) - {if (bbcp_Config.Options & bbcp_APPEND) totsz -= fp->targetsz; + {if (isAppend) totsz -= fp->targetsz; fp = fp->next; } + if (retc) return Request_exit(retc); -// Now determine if we need to reset the stat info on any paths we created -// - if ((fp = bbcp_Config.srcPath)) - {if (bbcp_Config.Options & bbcp_PCOPY) - {while(fp && fp->setStat(bbcp_Config.ModeD) == 0) fp = fp->next;} - else if (bbcp_Config.ModeD != bbcp_Config.ModeDC) - {while(fp && fp->setMode(bbcp_Config.ModeD) == 0) fp = fp->next;} - } +// Iterate through the copy set performing all required operations (recursive) +// + std::map::iterator it; + for (it = CopySet.begin(); it != CopySet.end(); ++it) + {dp = it->second; + if ((retc = dp->Create_Path())) return Request_exit(retc); + fp = dp->next; + while(fp) + {if (fp->Info.Otype == 'l') + {if ((retc = fp->Create_Link())) return Request_exit(retc); + } else { + if ((retc = Request_get(fp)) + || (retc = Local->RecvFile(fp,Remote))) + return Request_exit(retc); + if (isAppend) totsz -= fp->targetsz; + } + fp = fp->next; + } + if (doPcopy) dp->setStat(bbcp_Cfg.ModeD); + else if (setDMode) dp->setMode(bbcp_Cfg.ModeD); + } // Report back how many files and bytes were received // @@ -677,53 +745,67 @@ int bbcp_Protocol::Request_exit(int retc, const char *dRM) int bbcp_Protocol::Request_flist(long long &totsz, int &numlinks, bool dotrim) { + typedef std::pair Pair; + std::map::iterator it; int retc, noteol, numfiles = 0; - char *lp, *tfn;; + char *lp, *tfn, *slash; int tdln = strlen(tdir); - bbcp_FileSpec *fp, *lastfp = 0, *lastdp = 0, *lastsp = 0; + bbcp_FileSpec *fp; const char flcmd[] = "flist\n"; const int flcsz = sizeof(flcmd)-1; + bool orphan; // Set correct target file name // - if (bbcp_Config.snkSpec->Info.Otype == 'd') tfn = 0; - else tfn = bbcp_Config.snkSpec->filename; + if (bbcp_Cfg.snkSpec->Info.Otype == 'd') tfn = 0; + else tfn = bbcp_Cfg.snkSpec->filename; // Send request // if (Remote->Put(flcmd, flcsz)) return -1; -// Now start getting all of the files to be copied +// Now start getting all of the objects to be recreated here // numlinks = 0; while((lp = Remote->GetLine()) && (noteol = strcmp(lp, "eol"))) {fp = new bbcp_FileSpec(fs_obj, Remote->NodeName()); -// cerr <<"Get flist: " <Decode(lp)) {numfiles = -1; break;} if (fp->Compose(tdir_id, tdir, tdln, tfn) && (retc = fp->Xfr_Done())) {delete fp; if (retc < 0) {numfiles = -1; break;}} else if (fp->Info.Otype == 'd') - {if (dotrim) fp->setTrim(); - else {if (lastdp) lastdp->next = fp; - else bbcp_Config.srcPath = fp; - lastdp = fp; - } - } - else if (fp->Info.Otype == 'l') - {if (lastsp) lastsp->next = fp; - else bbcp_Config.slkPath = fp; - lastsp = fp; numlinks++; - } -/*PIPE*/ else if (fp->Info.Otype == 'f' || fp->Info.Otype == 'p') - {numfiles++; - totsz += fp->Info.size; - if (lastfp) lastfp->next = fp; - else bbcp_Config.srcSpec = fp; - lastfp = fp; + {if (dotrim) {fp->setTrim(); dotrim = false;} + else CopySet.insert(Pair(fp->pathname,fp)); } - dotrim = false; + else {dotrim = false; + switch(fp->Info.Otype) + {case 'f': numfiles++; + totsz += fp->Info.size; + break; + case 'l': numlinks++; + break; + case 'p': numfiles++; + break; + default: delete fp; + continue; + } + if (!(slash = rindex(fp->pathname, '/'))) orphan = true; + else {*slash = 0; + it = CopySet.find(fp->pathname); + *slash = '/'; + if (it == CopySet.end()) orphan = true; + else {bbcp_FileSpec *gp = it->second; + fp->next = gp->next; + gp->next = fp; + orphan = false; + } + } + if (orphan) + {fp->next = bbcp_Cfg.srcSpec; + bbcp_Cfg.srcSpec = fp; + } + } } // Flush the input queue if need be @@ -752,7 +834,7 @@ int bbcp_Protocol::Request_get(bbcp_FileSpec *fp) // Make sure there is enough space for this file // - if (!(bbcp_Config.Options & bbcp_NOSPCHK) && !(fs_obj->Enough(fp->Info.size,1))) + if (!(bbcp_Cfg.Options & bbcp_NOSPCHK) && !(fs_obj->Enough(fp->Info.size,1))) {bbcp_Fmsg("Request_get", "Insufficient space to create file", fp->targpath); return 28; @@ -798,9 +880,9 @@ int bbcp_Protocol::Request_login(bbcp_Link *Net) // Prepare the login request // - blen = sprintf(buff,id,bbcp_Config.SecToken,(bbcp_Net.AutoTune() ? "+" : ""), - bbcp_Config.Wsize, bbcp_Version.VData, - bbcp_Config.RWBsz); + blen = sprintf(buff,id,bbcp_Cfg.SecToken,(bbcp_Net.AutoTune() ? "+" : ""), + bbcp_Cfg.Wsize, bbcp_Version.VData, + bbcp_Cfg.RWBsz); // Send the request // @@ -841,7 +923,7 @@ int bbcp_Protocol::Request_login(bbcp_Link *Net) int bbcp_Protocol::AdjustWS(char *wp, char *bp, int Final) { int xWS, xAT, sWS, sAT, tWS, tAT, xBS = 0; - int isSRC = bbcp_Config.Options & bbcp_SRC; + int isSRC = bbcp_Cfg.Options & bbcp_SRC; // New version tell us if they can auto-tune via a leading plus // @@ -849,13 +931,13 @@ int bbcp_Protocol::AdjustWS(char *wp, char *bp, int Final) // Get window size in binary // - if (bbcp_Config.a2n("window size", wp, xWS, 1, -1)) return 0; + if (bbcp_Cfg.a2n("window size", wp, xWS, 1, -1)) return 0; // New versions also tell us the I/O buffer size as well. Older versions // do not give a buffer size and it was historically the window size. // if (bp && *bp) - {if (bbcp_Config.a2n("buffer size", bp, xBS, 1, -1)) return 0;} + {if (bbcp_Cfg.a2n("buffer size", bp, xBS, 1, -1)) return 0;} else xBS = xWS; // If this is a login response and the value is the window that must be used. @@ -863,14 +945,14 @@ int bbcp_Protocol::AdjustWS(char *wp, char *bp, int Final) // if (Final) {if (isSRC) - {if (xWS < bbcp_Config.Wsize) - {bbcp_Config.Wsize = bbcp_Net.setWindow(xWS, 1); - if (bbcp_Config.Options & bbcp_VERBOSE) - bbcp_Config.WAMsg("Logon","Source window size reduced",xWS); + {if (xWS < bbcp_Cfg.Wsize) + {bbcp_Cfg.Wsize = bbcp_Net.setWindow(xWS, 1); + if (bbcp_Cfg.Options & bbcp_VERBOSE) + bbcp_Cfg.WAMsg("Logon","Source window size reduced",xWS); } - if (xBS < bbcp_Config.RWBsz) bbcp_Config.setRWB(xBS); + if (xBS < bbcp_Cfg.RWBsz) bbcp_Cfg.setRWB(xBS); } else - if (xBS > bbcp_Config.RWBsz) bbcp_Config.setRWB(xBS); + if (xBS > bbcp_Cfg.RWBsz) bbcp_Cfg.setRWB(xBS); return 1; } @@ -880,21 +962,21 @@ int bbcp_Protocol::AdjustWS(char *wp, char *bp, int Final) // target or the traget buffer to be no smaller than the source buffer. // if (isSRC) - {if (xBS < bbcp_Config.RWBsz) bbcp_Config.setRWB(xBS); - sWS = bbcp_Config.Wsize; sAT = bbcp_Net.AutoTune(); + {if (xBS < bbcp_Cfg.RWBsz) bbcp_Cfg.setRWB(xBS); + sWS = bbcp_Cfg.Wsize; sAT = bbcp_Net.AutoTune(); tWS = xWS; tAT = xAT; } else { - if (xBS > bbcp_Config.RWBsz) bbcp_Config.setRWB(xBS); + if (xBS > bbcp_Cfg.RWBsz) bbcp_Cfg.setRWB(xBS); sWS = xWS; sAT = xAT; - tWS = bbcp_Config.Wsize; tAT = bbcp_Net.AutoTune(); + tWS = bbcp_Cfg.Wsize; tAT = bbcp_Net.AutoTune(); } // If source and target autotune then check if the target window >= source (warn // o/w) and return back what the sender would have expected to keep autotuning. // if (sAT && tAT) - {if (tWS < sWS && bbcp_Config.Options & bbcp_VERBOSE) - bbcp_Config.WAMsg("Login","Target autotuning may be misconfigured; " + {if (tWS < sWS && bbcp_Cfg.Options & bbcp_VERBOSE) + bbcp_Cfg.WAMsg("Login","Target autotuning may be misconfigured; " "max set", tWS); return (isSRC ? tWS : sWS); } @@ -903,9 +985,9 @@ int bbcp_Protocol::AdjustWS(char *wp, char *bp, int Final) // if (isSRC) {if (sWS > tWS) - {bbcp_Config.Wsize = bbcp_Net.setWindow(tWS, 1); - if (bbcp_Config.Options & bbcp_VERBOSE) - bbcp_Config.WAMsg("Login", "Source window size reduced", tWS); + {bbcp_Cfg.Wsize = bbcp_Net.setWindow(tWS, 1); + if (bbcp_Cfg.Options & bbcp_VERBOSE) + bbcp_Cfg.WAMsg("Login", "Source window size reduced", tWS); } return tWS; } @@ -916,6 +998,32 @@ int bbcp_Protocol::AdjustWS(char *wp, char *bp, int Final) return xWS; } +/******************************************************************************/ +/* C h k 4 R e d i r */ +/******************************************************************************/ + +void bbcp_Protocol::Chk4Redir(const char *oldHost, const char *newHost) +{ + bbcp_NetAddr oldAddr; + bbcp_NetAddr newAddr; + char buff[128]; + int n; + +// Establish addresses and check if they are the same (port doesn't matter) +// + oldAddr.Set(oldHost,0); + newAddr.Set(newHost,0); + if (oldAddr.Same(&newAddr)) return; + +// Get information to blab out +// + buff[0] = '('; + n = newAddr.Format(buff, sizeof(buff)-8, bbcp_NetAddrInfo::fmtAddr, + bbcp_NetAddrInfo::noPort); + strcpy(&buff[n+1],")."); + bbcp_Fmsg("Protocol","Host",oldHost,"redirect connection to",newHost,buff); +} + /******************************************************************************/ /* p u t C S V */ /******************************************************************************/ @@ -923,8 +1031,8 @@ int bbcp_Protocol::AdjustWS(char *wp, char *bp, int Final) void bbcp_Protocol::putCSV(char *Host, char *csFn, char *csVal, int csVsz) { //1234567890123 struct iovec iov[] = {{(char *)"Checksum: ", 10}, - {bbcp_Config.csName,strlen(bbcp_Config.csName)}, - {(char *)" ", 1}, {csVal, csVsz}, + {bbcp_Cfg.csName,strlen(bbcp_Cfg.csName)}, + {(char *)" ", 1}, {csVal, static_cast(csVsz)}, {(char *)" ", 1}, {Host, strlen(Host)}, {(char *)":", 1}, {csFn, strlen(csFn)}, {(char *)"\n",1}}; @@ -932,10 +1040,10 @@ void bbcp_Protocol::putCSV(char *Host, char *csFn, char *csVal, int csVsz) // Write the checksum to a special file if it exists // - if (bbcp_Config.csPath) - {if (writev(bbcp_Config.csFD, iov, n) < 0) - {bbcp_Emsg("Protocol",errno,"writing checksum to",bbcp_Config.csPath); - close(bbcp_Config.csFD); bbcp_Config.csFD = -1; + if (bbcp_Cfg.csPath) + {if (writev(bbcp_Cfg.csFD, iov, n) < 0) + {bbcp_Emsg("Protocol",errno,"writing checksum to",bbcp_Cfg.csPath); + close(bbcp_Cfg.csFD); bbcp_Cfg.csFD = -1; } } else writev(STDERR_FILENO, iov, n); } @@ -952,9 +1060,9 @@ int bbcp_Protocol::SendArgs(bbcp_Node *Node, bbcp_FileSpec *fsp, // The remote program should be running at this point, setup the args // - if (bbcp_Config.CopyOpts) - {apnt[i] = bbcp_Config.CopyOpts; - alen[i++] = strlen(bbcp_Config.CopyOpts); + if (bbcp_Cfg.CopyOpts) + {apnt[i] = bbcp_Cfg.CopyOpts; + alen[i++] = strlen(bbcp_Cfg.CopyOpts); } if (addOpt) {apnt[i] = addOpt; alen[i++] = strlen(addOpt);} apnt[i] = buff; diff --git a/src/bbcp_Protocol.h b/src/bbcp_Protocol.h index ac87e25..fe50659 100644 --- a/src/bbcp_Protocol.h +++ b/src/bbcp_Protocol.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ P r o t o c o l . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -67,7 +68,8 @@ char *tdir; long long tdir_id; int AdjustWS(char *wp, char *bp, int Final); -int getCBPort(bbcp_Node *Node); +void Chk4Redir(const char *oldHost, const char *newHost); +int getCBPort(bbcp_Node *Node, char *&hDest); void getEnd(bbcp_Node *Node); void putCSV(char *Host, char *csFn, char *csVal, int csVsz); diff --git a/src/bbcp_Pthread.C b/src/bbcp_Pthread.C index c6c909b..39f8539 100644 --- a/src/bbcp_Pthread.C +++ b/src/bbcp_Pthread.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ P t h r e a d . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Pthread.h b/src/bbcp_Pthread.h index a5eb040..142700e 100644 --- a/src/bbcp_Pthread.h +++ b/src/bbcp_Pthread.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ P t h r e a d . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -32,6 +33,8 @@ #include #include +#include + class bbcp_CondVar { public: @@ -155,7 +158,7 @@ inline void Wait() {int rc; bbcp_Semaphore(int semval=1) {if (sem_init(&h_semaphore, 0, semval)) {throw "sem_init() failed", errno;} } - ~bbcp_Semaphore() {if (sem_destroy(&h_semaphore)) + ~bbcp_Semaphore() throw(std::exception) {if (sem_destroy(&h_semaphore)) {throw "sem_destroy() failed", errno;} } diff --git a/src/bbcp_RTCopy.C b/src/bbcp_RTCopy.C index 9a22313..049ddf5 100644 --- a/src/bbcp_RTCopy.C +++ b/src/bbcp_RTCopy.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ R T C o p y . C */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -49,7 +50,7 @@ /* G l o b a l V a l u e s */ /******************************************************************************/ -extern bbcp_Config bbcp_Config; +extern bbcp_Config bbcp_Cfg; bbcp_RTCopy bbcp_RTCopy; @@ -102,7 +103,7 @@ void *bbcp_RTCopy::Lock(bbcp_Semaphore *semP) // Check if we should verify this copy // - if (Grow && bbcp_Config.Options & bbcp_RTCVERC) + if (Grow && bbcp_Cfg.Options & bbcp_RTCVERC) {lfSize = FSp->getSize(lkFD); if (lfSize < 0) {rc = static_cast(-lfSize); @@ -150,7 +151,7 @@ long long bbcp_RTCopy::Prep(long long rdsk, int rdsz, ssize_t &rlen) // Setup high watermark and the time limit, if any // Hwm = rdsk + rdsz; - tVal = bbcp_Config.rtLimit; + tVal = bbcp_Cfg.rtLimit; pSize = -1; // We continue waiting for enough data to fully satisfy this read or until @@ -162,10 +163,10 @@ long long bbcp_RTCopy::Prep(long long rdsk, int rdsz, ssize_t &rlen) {Left = cSize - Hwm; return (size_t)0x7fffffff; } - sleep(bbcp_Config.rtCheck); + sleep(bbcp_Cfg.rtCheck); if (tVal) {if (cSize != pSize) {tLim = tVal; pSize = cSize;} - else if ((tLim -= bbcp_Config.rtCheck) <= 0) + else if ((tLim -= bbcp_Cfg.rtCheck) <= 0) {rlen = -ETIMEDOUT; return -1;} } } @@ -193,14 +194,14 @@ int bbcp_RTCopy::Start(bbcp_FileSystem *fsp, const char *iofn, int iofd) ioFD = iofd; Grow = 1; Left = 0; - Blok = (bbcp_Config.Options & bbcp_RTCBLOK ? bbcp_Config.Streams : 0); + Blok = (bbcp_Cfg.Options & bbcp_RTCBLOK ? bbcp_Cfg.Streams : 0); FSp = fsp; // Initialize variable dependent on how we will do locking // - if (bbcp_Config.rtLockf) - {lkFN = bbcp_Config.rtLockf; - lkFD = bbcp_Config.rtLockd; + if (bbcp_Cfg.rtLockf) + {lkFN = bbcp_Cfg.rtLockf; + lkFD = bbcp_Cfg.rtLockd; } else { lkFN = iofn; lkFD = dup(iofd); diff --git a/src/bbcp_RTCopy.h b/src/bbcp_RTCopy.h index d733089..9cc88ed 100644 --- a/src/bbcp_RTCopy.h +++ b/src/bbcp_RTCopy.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ R T C o p y . h */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Set.C b/src/bbcp_Set.C index 37264d1..2662445 100644 --- a/src/bbcp_Set.C +++ b/src/bbcp_Set.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ S e t . C */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Set.h b/src/bbcp_Set.h index 4d5591e..9ed96c2 100644 --- a/src/bbcp_Set.h +++ b/src/bbcp_Set.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ S e t . h */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Stream.C b/src/bbcp_Stream.C index bb4fb7d..1f4c655 100644 --- a/src/bbcp_Stream.C +++ b/src/bbcp_Stream.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ S t r e a m . c */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ diff --git a/src/bbcp_Stream.h b/src/bbcp_Stream.h index bbeb3a4..d62326d 100644 --- a/src/bbcp_Stream.h +++ b/src/bbcp_Stream.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ S t r e a m . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Deprtment of Energy */ /* */ diff --git a/src/bbcp_System.C b/src/bbcp_System.C index 4be94bf..b5059fc 100644 --- a/src/bbcp_System.C +++ b/src/bbcp_System.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ S y s t e m . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -60,9 +61,6 @@ bbcp_System bbcp_OS; -extern bbcp_Config bbcp_Config; - - /******************************************************************************/ /* C o n s t r u c t o r */ /******************************************************************************/ diff --git a/src/bbcp_System.h b/src/bbcp_System.h index 06e17e5..cb440ef 100644 --- a/src/bbcp_System.h +++ b/src/bbcp_System.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ S y s t e m . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Timer.C b/src/bbcp_Timer.C index f158cb1..260a3ad 100644 --- a/src/bbcp_Timer.C +++ b/src/bbcp_Timer.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ T i m e r . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Timer.h b/src/bbcp_Timer.h index 48c277f..c4970cf 100644 --- a/src/bbcp_Timer.h +++ b/src/bbcp_Timer.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ T i m e r . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_Version.C b/src/bbcp_Version.C index 7c4a345..d888ff8 100644 --- a/src/bbcp_Version.C +++ b/src/bbcp_Version.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ V e r s i o n . C */ /* */ -/*(c) 2010-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. Scroll to end for Terms and Conditions of use */ +/*(c) 2010-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. Scroll to end for Terms and Conditions of use */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ @@ -37,7 +38,7 @@ /* */ /******************************************************************************/ -extern bbcp_Config bbcp_Config; +extern bbcp_Config bbcp_Cfg; bbcp_Version bbcp_Version; @@ -47,9 +48,10 @@ extern bbcp_Config bbcp_Config; bbcp_Version::bbcp_Version() { -Copyright = "(c) 2002-2015 by the Board of Trustees of the Leland Stanford, Jr., University"; +Copyright = "(c) 2002-2017 by the Board of Trustees of the Leland Stanford, Jr., University"; Author = "Andrew Hanushevsky"; -Version = "Version: 15.02.03.01.1"; +// 12345678901 +Version = "Version: 17.12.00.00.0"; // 0123456789 VData = Version+9; }; @@ -78,9 +80,9 @@ int bbcp_Version::Verify(char *node, char *nodever) bbcp_Fmsg("Version", "Warning:",node,"is running",vset,"version of bbcp"); - if (bbcp_Config.Options & bbcp_BLAB) + if (bbcp_Cfg.Options & bbcp_BLAB) {bbcp_Fmsg("Version", node, "running version", nodever); - bbcp_Fmsg("Version", bbcp_Config.MyHost, "running version", VData); + bbcp_Fmsg("Version", bbcp_Cfg.MyHost, "running version", VData); } return 0; }; diff --git a/src/bbcp_Version.h b/src/bbcp_Version.h index fcfc29c..fcc6f4f 100644 --- a/src/bbcp_Version.h +++ b/src/bbcp_Version.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ V e r s i o n . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_ZCX.C b/src/bbcp_ZCX.C index 7b01c4b..7331698 100644 --- a/src/bbcp_ZCX.C +++ b/src/bbcp_ZCX.C @@ -2,7 +2,8 @@ /* */ /* b b c p _ Z C X . C */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */ diff --git a/src/bbcp_ZCX.h b/src/bbcp_ZCX.h index 7105c01..8f30a2a 100644 --- a/src/bbcp_ZCX.h +++ b/src/bbcp_ZCX.h @@ -4,7 +4,8 @@ /* */ /* b b c p _ Z C X . h */ /* */ -/*(c) 2002-14 by the Board of Trustees of the Leland Stanford, Jr., University*//* All Rights Reserved. See bbcp_Version.C for complete License Terms *//* All Rights Reserved */ +/*(c) 2002-17 by the Board of Trustees of the Leland Stanford, Jr., University*/ +/* All Rights Reserved. See bbcp_Version.C for complete License Terms */ /* Produced by Andrew Hanushevsky for Stanford University under contract */ /* DE-AC02-76-SFO0515 with the Department of Energy */ /* */