nbd-tester-client.c

   1 /*
   2  * Test client to test the NBD server. Doesn't do anything useful, except
   3  * checking that the server does, actually, work.
   4  *
   5  * Note that the only 'real' test is to check the client against a kernel. If
   6  * it works here but does not work in the kernel, then that's most likely a bug
   7  * in this program and/or in nbd-server.
   8  *
   9  * Copyright(c) 2006  Wouter Verhelst
  10  *
  11  * This program is Free Software; you can redistribute it and/or modify it
  12  * under the terms of the GNU General Public License as published by the Free
  13  * Software Foundation, in version 2.
  14  *
  15  * This program is distributed in the hope that it will be useful, but WITHOUT
  16  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  17  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  18  * more details.
  19  *
  20  * You should have received a copy of the GNU General Public License along with
  21  * this program; if not, write to the Free Software Foundation, Inc., 51
  22  * Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  23  */
  24 #include <stdlib.h>
  25 #include <stdio.h>
  26 #include <stdbool.h>
  27 #include <string.h>
  28 #include <sys/time.h>
  29 #include <sys/types.h>
  30 #include <sys/socket.h>
  31 #include <sys/stat.h>
  32 #include <sys/mman.h>
  33 #include <fcntl.h>
  34 #include <syslog.h>
  35 #include <unistd.h>
  36 #include "config.h"
  37 #include "lfs.h"
  38 #include <netinet/in.h>
  39 #include <glib.h>
  40
  41 #define MY_NAME "nbd-tester-client"
  42 #include "cliserv.h"
  43
  44 static gchar errstr[1024];
  45 const static int errstr_len=1024;
  46
  47 static uint64_t size;
  48
  49 static gchar * transactionlog = "nbd-tester-client.tr";
  50
  51 typedef enum {
  52         CONNECTION_TYPE_NONE,
  53         CONNECTION_TYPE_CONNECT,
  54         CONNECTION_TYPE_INIT_PASSWD,
  55         CONNECTION_TYPE_CLISERV,
  56         CONNECTION_TYPE_FULL,
  57 } CONNECTION_TYPE;
  58
  59 typedef enum {
  60         CONNECTION_CLOSE_PROPERLY,
  61         CONNECTION_CLOSE_FAST,
  62 } CLOSE_TYPE;
  63
  64 struct reqcontext {
  65         uint64_t seq;
  66         struct nbd_request req;
  67         struct reqcontext * next;
  68         struct reqcontext * prev;
  69 };
  70
  71 struct rclist {
  72         struct reqcontext * head;
  73         struct reqcontext * tail;
  74         int numitems;
  75 };
  76
  77 void rclist_unlink(struct rclist * l, struct reqcontext * p) {
  78         if (p && l) {
  79                 struct reqcontext * prev = p->prev;
  80                 struct reqcontext * next = p->next;
  81
  82                 /* Fix link to previous */
  83                 if (prev)
  84                         prev->next = next;
  85                 else
  86                         l->head = next;
  87
  88                 if (next)
  89                         next->prev = prev;
  90                 else
  91                         l->tail = prev;
  92
  93                 p->prev = NULL;
  94                 p->next = NULL;
  95                 l->numitems--;
  96         }
  97 }
  98
  99 /* Add a new list item to the tail */
 100 void rclist_addtail(struct rclist * l, struct reqcontext * p)
 101 {
 102         if (!p || !l)
 103                 return;
 104         if (l->tail) {
 105                 if (l->tail->next)
 106                         g_warning("addtail found list tail has a next pointer");
 107                 l->tail->next = p;
 108                 p->next = NULL;
 109                 p->prev = l->tail;
 110                 l->tail = p;
 111         } else {
 112                 if (l->head)
 113                         g_warning("addtail found no list tail but a list head");
 114                 l->head = p;
 115                 l->tail = p;
 116                 p->prev = NULL;
 117                 p->next = NULL;
 118         }
 119         l->numitems++;
 120 }
 121
 122 #define TEST_WRITE (1<<0)
 123 #define TEST_FLUSH (1<<1)
 124
 125 int timeval_subtract (struct timeval *result, struct timeval *x,
 126                       struct timeval *y) {
 127         if (x->tv_usec < y->tv_usec) {
 128                 int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1;
 129                 y->tv_usec -= 1000000 * nsec;
 130                 y->tv_sec += nsec;
 131         }
 132
 133         if (x->tv_usec - y->tv_usec > 1000000) {
 134                 int nsec = (x->tv_usec - y->tv_usec) / 1000000;
 135                 y->tv_usec += 1000000 * nsec;
 136                 y->tv_sec -= nsec;
 137         }
 138
 139         result->tv_sec = x->tv_sec - y->tv_sec;
 140         result->tv_usec = x->tv_usec - y->tv_usec;
 141
 142         return x->tv_sec < y->tv_sec;
 143 }
 144
 145 double timeval_diff_to_double (struct timeval * x, struct timeval * y) {
 146         struct timeval r;
 147         timeval_subtract(&r, x, y);
 148         return r.tv_sec * 1.0 + r.tv_usec/1000000.0;
 149 }
 150
 151 static inline int read_all(int f, void *buf, size_t len) {
 152         ssize_t res;
 153         size_t retval=0;
 154
 155         while(len>0) {
 156                 if((res=read(f, buf, len)) <=0) {
 157                         if (!res)
 158                                 errno=EAGAIN;
 159                         snprintf(errstr, errstr_len, "Read failed: %s", strerror(errno));
 160                         return -1;
 161                 }
 162                 len-=res;
 163                 buf+=res;
 164                 retval+=res;
 165         }
 166         return retval;
 167 }
 168
 169 static inline int write_all(int f, void *buf, size_t len) {
 170         ssize_t res;
 171         size_t retval=0;
 172
 173         while(len>0) {
 174                 if((res=write(f, buf, len)) <=0) {
 175                         if (!res)
 176                                 errno=EAGAIN;
 177                         snprintf(errstr, errstr_len, "Write failed: %s", strerror(errno));
 178                         return -1;
 179                 }
 180                 len-=res;
 181                 buf+=res;
 182                 retval+=res;
 183         }
 184         return retval;
 185 }
 186
 187 #define READ_ALL_ERRCHK(f, buf, len, whereto, errmsg...) if((read_all(f, buf, len))<=0) { snprintf(errstr, errstr_len, ##errmsg); goto whereto; }
 188 #define READ_ALL_ERR_RT(f, buf, len, whereto, rval, errmsg...) if((read_all(f, buf, len))<=0) { snprintf(errstr, errstr_len, ##errmsg); retval = rval; goto whereto; }
 189
 190 #define WRITE_ALL_ERRCHK(f, buf, len, whereto, errmsg...) if((write_all(f, buf, len))<=0) { snprintf(errstr, errstr_len, ##errmsg); goto whereto; }
 191 #define WRITE_ALL_ERR_RT(f, buf, len, whereto, rval, errmsg...) if((write_all(f, buf, len))<=0) { snprintf(errstr, errstr_len, ##errmsg); retval = rval; goto whereto; }
 192
 193 int setup_connection(gchar *hostname, int port, gchar* name, CONNECTION_TYPE ctype, int* serverflags) {
 194         int sock;
 195         struct hostent *host;
 196         struct sockaddr_in addr;
 197         char buf[256];
 198         uint64_t mymagic = (name ? opts_magic : cliserv_magic);
 199         u64 tmp64;
 200         uint32_t tmp32 = 0;
 201
 202         sock=0;
 203         if(ctype<CONNECTION_TYPE_CONNECT)
 204                 goto end;
 205         if((sock=socket(PF_INET, SOCK_STREAM, IPPROTO_TCP))<0) {
 206                 strncpy(errstr, strerror(errno), errstr_len);
 207                 goto err;
 208         }
 209         setmysockopt(sock);
 210         if(!(host=gethostbyname(hostname))) {
 211                 strncpy(errstr, strerror(errno), errstr_len);
 212                 goto err_open;
 213         }
 214         addr.sin_family=AF_INET;
 215         addr.sin_port=htons(port);
 216         addr.sin_addr.s_addr=*((int *) host->h_addr);
 217         if((connect(sock, (struct sockaddr *)&addr, sizeof(addr))<0)) {
 218                 strncpy(errstr, strerror(errno), errstr_len);
 219                 goto err_open;
 220         }
 221         if(ctype<CONNECTION_TYPE_INIT_PASSWD)
 222                 goto end;
 223         READ_ALL_ERRCHK(sock, buf, strlen(INIT_PASSWD), err_open, "Could not read INIT_PASSWD: %s", strerror(errno));
 224         if(strlen(buf)==0) {
 225                 snprintf(errstr, errstr_len, "Server closed connection");
 226                 goto err_open;
 227         }
 228         if(strncmp(buf, INIT_PASSWD, strlen(INIT_PASSWD))) {
 229                 snprintf(errstr, errstr_len, "INIT_PASSWD does not match");
 230                 goto err_open;
 231         }
 232         if(ctype<CONNECTION_TYPE_CLISERV)
 233                 goto end;
 234         READ_ALL_ERRCHK(sock, &tmp64, sizeof(tmp64), err_open, "Could not read cliserv_magic: %s", strerror(errno));
 235         tmp64=ntohll(tmp64);
 236         if(tmp64 != mymagic) {
 237                 strncpy(errstr, "mymagic does not match", errstr_len);
 238                 goto err_open;
 239         }
 240         if(ctype<CONNECTION_TYPE_FULL)
 241                 goto end;
 242         if(!name) {
 243                 READ_ALL_ERRCHK(sock, &size, sizeof(size), err_open, "Could not read size: %s", strerror(errno));
 244                 size=ntohll(size);
 245                 READ_ALL_ERRCHK(sock, buf, 128, err_open, "Could not read data: %s", strerror(errno));
 246                 goto end;
 247         }
 248         /* flags */
 249         READ_ALL_ERRCHK(sock, buf, sizeof(uint16_t), err_open, "Could not read reserved field: %s", strerror(errno));
 250         /* reserved field */
 251         WRITE_ALL_ERRCHK(sock, &tmp32, sizeof(tmp32), err_open, "Could not write reserved field: %s", strerror(errno));
 252         /* magic */
 253         tmp64 = htonll(opts_magic);
 254         WRITE_ALL_ERRCHK(sock, &tmp64, sizeof(tmp64), err_open, "Could not write magic: %s", strerror(errno));
 255         /* name */
 256         tmp32 = htonl(NBD_OPT_EXPORT_NAME);
 257         WRITE_ALL_ERRCHK(sock, &tmp32, sizeof(tmp32), err_open, "Could not write option: %s", strerror(errno));
 258         tmp32 = htonl((uint32_t)strlen(name));
 259         WRITE_ALL_ERRCHK(sock, &tmp32, sizeof(tmp32), err_open, "Could not write name length: %s", strerror(errno));
 260         WRITE_ALL_ERRCHK(sock, name, strlen(name), err_open, "Could not write name:: %s", strerror(errno));
 261         READ_ALL_ERRCHK(sock, &size, sizeof(size), err_open, "Could not read size: %s", strerror(errno));
 262         size = ntohll(size);
 263         uint16_t flags;
 264         READ_ALL_ERRCHK(sock, &flags, sizeof(uint16_t), err_open, "Could not read flags: %s", strerror(errno));
 265         flags = ntohs(flags);
 266         *serverflags = flags;
 267         READ_ALL_ERRCHK(sock, buf, 124, err_open, "Could not read reserved zeroes: %s", strerror(errno));
 268         goto end;
 269 err_open:
 270         close(sock);
 271 err:
 272         sock=-1;
 273 end:
 274         return sock;
 275 }
 276
 277 int close_connection(int sock, CLOSE_TYPE type) {
 278         struct nbd_request req;
 279         u64 counter=0;
 280
 281         switch(type) {
 282                 case CONNECTION_CLOSE_PROPERLY:
 283                         req.magic=htonl(NBD_REQUEST_MAGIC);
 284                         req.type=htonl(NBD_CMD_DISC);
 285                         memcpy(&(req.handle), &(counter), sizeof(counter));
 286                         counter++;
 287                         req.from=0;
 288                         req.len=0;
 289                         if(write(sock, &req, sizeof(req))<0) {
 290                                 snprintf(errstr, errstr_len, "Could not write to socket: %s", strerror(errno));
 291                                 return -1;
 292                         }
 293                 case CONNECTION_CLOSE_FAST:
 294                         if(close(sock)<0) {
 295                                 snprintf(errstr, errstr_len, "Could not close socket: %s", strerror(errno));
 296                                 return -1;
 297                         }
 298                         break;
 299                 default:
 300                         g_critical("Your compiler is on crack!"); /* or I am buggy */
 301                         return -1;
 302         }
 303         return 0;
 304 }
 305
 306 int read_packet_check_header(int sock, size_t datasize, long long int curhandle) {
 307         struct nbd_reply rep;
 308         int retval=0;
 309         char buf[datasize];
 310
 311         READ_ALL_ERR_RT(sock, &rep, sizeof(rep), end, -1, "Could not read reply header: %s", strerror(errno));
 312         rep.magic=ntohl(rep.magic);
 313         rep.error=ntohl(rep.error);
 314         if(rep.magic!=NBD_REPLY_MAGIC) {
 315                 snprintf(errstr, errstr_len, "Received package with incorrect reply_magic. Index of sent packages is %lld (0x%llX), received handle is %lld (0x%llX). Received magic 0x%lX, expected 0x%lX", (long long int)curhandle, (long long unsigned int)curhandle, (long long int)*((u64*)rep.handle), (long long unsigned int)*((u64*)rep.handle), (long unsigned int)rep.magic, (long unsigned int)NBD_REPLY_MAGIC);
 316                 retval=-1;
 317                 goto end;
 318         }
 319         if(rep.error) {
 320                 snprintf(errstr, errstr_len, "Received error from server: %ld (0x%lX). Handle is %lld (0x%llX).", (long int)rep.error, (long unsigned int)rep.error, (long long int)(*((u64*)rep.handle)), (long long unsigned int)*((u64*)rep.handle));
 321                 retval=-1;
 322                 goto end;
 323         }
 324         if (datasize)
 325                 READ_ALL_ERR_RT(sock, &buf, datasize, end, -1, "Could not read data: %s", strerror(errno));
 326
 327 end:
 328         return retval;
 329 }
 330
 331 int oversize_test(gchar* hostname, int port, char* name, int sock,
 332                   char sock_is_open, char close_sock, int testflags) {
 333         int retval=0;
 334         struct nbd_request req;
 335         struct nbd_reply rep;
 336         int i=0;
 337         int serverflags = 0;
 338         pid_t G_GNUC_UNUSED mypid = getpid();
 339         char buf[((1024*1024)+sizeof(struct nbd_request)/2)<<1];
 340         bool got_err;
 341
 342         /* This should work */
 343         if(!sock_is_open) {
 344                 if((sock=setup_connection(hostname, port, name, CONNECTION_TYPE_FULL, &serverflags))<0) {
 345                         g_warning("Could not open socket: %s", errstr);
 346                         retval=-1;
 347                         goto err;
 348                 }
 349         }
 350         req.magic=htonl(NBD_REQUEST_MAGIC);
 351         req.type=htonl(NBD_CMD_READ);
 352         req.len=htonl(1024*1024);
 353         memcpy(&(req.handle),&i,sizeof(i));
 354         req.from=htonll(i);
 355         WRITE_ALL_ERR_RT(sock, &req, sizeof(req), err, -1, "Could not write request: %s", strerror(errno));
 356         printf("%d: testing oversized request: %d: ", getpid(), ntohl(req.len));
 357         READ_ALL_ERR_RT(sock, &rep, sizeof(struct nbd_reply), err, -1, "Could not read reply header: %s", strerror(errno));
 358         READ_ALL_ERR_RT(sock, &buf, ntohl(req.len), err, -1, "Could not read data: %s", strerror(errno));
 359         if(rep.error) {
 360                 snprintf(errstr, errstr_len, "Received unexpected error: %d", rep.error);
 361                 retval=-1;
 362                 goto err;
 363         } else {
 364                 printf("OK\n");
 365         }
 366         /* This probably should not work */
 367         i++; req.from=htonll(i);
 368         req.len = htonl(ntohl(req.len) + sizeof(struct nbd_request) / 2);
 369         WRITE_ALL_ERR_RT(sock, &req, sizeof(req), err, -1, "Could not write request: %s", strerror(errno));
 370         printf("%d: testing oversized request: %d: ", getpid(), ntohl(req.len));
 371         READ_ALL_ERR_RT(sock, &rep, sizeof(struct nbd_reply), err, -1, "Could not read reply header: %s", strerror(errno));
 372         READ_ALL_ERR_RT(sock, &buf, ntohl(req.len), err, -1, "Could not read data: %s", strerror(errno));
 373         if(rep.error) {
 374                 printf("Received expected error\n");
 375                 got_err=true;
 376         } else {
 377                 printf("OK\n");
 378                 got_err=false;
 379         }
 380         /* ... unless this works, too */
 381         i++; req.from=htonll(i);
 382         req.len = htonl(ntohl(req.len) << 1);
 383         WRITE_ALL_ERR_RT(sock, &req, sizeof(req), err, -1, "Could not write request: %s", strerror(errno));
 384         printf("%d: testing oversized request: %d: ", getpid(), ntohl(req.len));
 385         READ_ALL_ERR_RT(sock, &rep, sizeof(struct nbd_reply), err, -1, "Could not read reply header: %s", strerror(errno));
 386         READ_ALL_ERR_RT(sock, &buf, ntohl(req.len), err, -1, "Could not read data: %s", strerror(errno));
 387         if(rep.error) {
 388                 printf("error\n");
 389         } else {
 390                 printf("OK\n");
 391         }
 392         if((rep.error && !got_err) || (!rep.error && got_err)) {
 393                 printf("Received unexpected error\n");
 394                 retval=-1;
 395         }
 396   err:
 397         return retval;
 398 }
 399
 400 int throughput_test(gchar* hostname, int port, char* name, int sock,
 401                     char sock_is_open, char close_sock, int testflags) {
 402         long long int i;
 403         char writebuf[1024];
 404         struct nbd_request req;
 405         int requests=0;
 406         fd_set set;
 407         struct timeval tv;
 408         struct timeval start;
 409         struct timeval stop;
 410         double timespan;
 411         double speed;
 412         char speedchar[2] = { '\0', '\0' };
 413         int retval=0;
 414         int serverflags = 0;
 415         signed int do_write=TRUE;
 416         pid_t mypid = getpid();
 417
 418
 419         if (!(testflags & TEST_WRITE))
 420                 testflags &= ~TEST_FLUSH;
 421
 422         memset (writebuf, 'X', 1024);
 423         size=0;
 424         if(!sock_is_open) {
 425                 if((sock=setup_connection(hostname, port, name, CONNECTION_TYPE_FULL, &serverflags))<0) {
 426                         g_warning("Could not open socket: %s", errstr);
 427                         retval=-1;
 428                         goto err;
 429                 }
 430         }
 431         if ((testflags & TEST_FLUSH) && ((serverflags & (NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA))
 432                                          != (NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA))) {
 433                 snprintf(errstr, errstr_len, "Server did not supply flush capability flags");
 434                 retval = -1;
 435                 goto err_open;
 436         }
 437         req.magic=htonl(NBD_REQUEST_MAGIC);
 438         req.len=htonl(1024);
 439         if(gettimeofday(&start, NULL)<0) {
 440                 retval=-1;
 441                 snprintf(errstr, errstr_len, "Could not measure start time: %s", strerror(errno));
 442                 goto err_open;
 443         }
 444         for(i=0;i+1024<=size;i+=1024) {
 445                 if(do_write) {
 446                         int sendfua = (testflags & TEST_FLUSH) && (((i>>10) & 15) == 3);
 447                         int sendflush = (testflags & TEST_FLUSH) && (((i>>10) & 15) == 11);
 448                         req.type=htonl((testflags & TEST_WRITE)?NBD_CMD_WRITE:NBD_CMD_READ);
 449                         if (sendfua)
 450                                 req.type = htonl(NBD_CMD_WRITE | NBD_CMD_FLAG_FUA);
 451                         memcpy(&(req.handle),&i,sizeof(i));
 452                         req.from=htonll(i);
 453                         if (write_all(sock, &req, sizeof(req)) <0) {
 454                                 retval=-1;
 455                                 goto err_open;
 456                         }
 457                         if (testflags & TEST_WRITE) {
 458                                 if (write_all(sock, writebuf, 1024) <0) {
 459                                         retval=-1;
 460                                         goto err_open;
 461                                 }
 462                         }
 463                         printf("%d: Requests(+): %d\n", (int)mypid, ++requests);
 464                         if (sendflush) {
 465                                 long long int j = i ^ (1LL<<63);
 466                                 req.type = htonl(NBD_CMD_FLUSH);
 467                                 memcpy(&(req.handle),&j,sizeof(j));
 468                                 req.from=0;
 469                                 if (write_all(sock, &req, sizeof(req)) <0) {
 470                                         retval=-1;
 471                                         goto err_open;
 472                                 }
 473                                 printf("%d: Requests(+): %d\n", (int)mypid, ++requests);
 474                         }
 475                 }
 476                 do {
 477                         FD_ZERO(&set);
 478                         FD_SET(sock, &set);
 479                         tv.tv_sec=0;
 480                         tv.tv_usec=0;
 481                         select(sock+1, &set, NULL, NULL, &tv);
 482                         if(FD_ISSET(sock, &set)) {
 483                                 /* Okay, there's something ready for
 484                                  * reading here */
 485                                 if(read_packet_check_header(sock, (testflags & TEST_WRITE)?0:1024, i)<0) {
 486                                         retval=-1;
 487                                         goto err_open;
 488                                 }
 489                                 printf("%d: Requests(-): %d\n", (int)mypid, --requests);
 490                         }
 491                 } while FD_ISSET(sock, &set);
 492                 /* Now wait until we can write again or until a second have
 493                  * passed, whichever comes first*/
 494                 FD_ZERO(&set);
 495                 FD_SET(sock, &set);
 496                 tv.tv_sec=1;
 497                 tv.tv_usec=0;
 498                 do_write=select(sock+1,NULL,&set,NULL,&tv);
 499                 if(!do_write) printf("Select finished\n");
 500                 if(do_write<0) {
 501                         snprintf(errstr, errstr_len, "select: %s", strerror(errno));
 502                         retval=-1;
 503                         goto err_open;
 504                 }
 505         }
 506         /* Now empty the read buffer */
 507         do {
 508                 FD_ZERO(&set);
 509                 FD_SET(sock, &set);
 510                 tv.tv_sec=0;
 511                 tv.tv_usec=0;
 512                 select(sock+1, &set, NULL, NULL, &tv);
 513                 if(FD_ISSET(sock, &set)) {
 514                         /* Okay, there's something ready for
 515                          * reading here */
 516                         read_packet_check_header(sock, (testflags & TEST_WRITE)?0:1024, i);
 517                         printf("%d: Requests(-): %d\n", (int)mypid, --requests);
 518                 }
 519         } while (requests);
 520         if(gettimeofday(&stop, NULL)<0) {
 521                 retval=-1;
 522                 snprintf(errstr, errstr_len, "Could not measure end time: %s", strerror(errno));
 523                 goto err_open;
 524         }
 525         timespan=timeval_diff_to_double(&stop, &start);
 526         speed=size/timespan;
 527         if(speed>1024) {
 528                 speed=speed/1024.0;
 529                 speedchar[0]='K';
 530         }
 531         if(speed>1024) {
 532                 speed=speed/1024.0;
 533                 speedchar[0]='M';
 534         }
 535         if(speed>1024) {
 536                 speed=speed/1024.0;
 537                 speedchar[0]='G';
 538         }
 539         g_message("%d: Throughput %s test (%s flushes) complete. Took %.3f seconds to complete, %.3f%sib/s", (int)getpid(), (testflags & TEST_WRITE)?"write":"read", (testflags & TEST_FLUSH)?"with":"without", timespan, speed, speedchar);
 540
 541 err_open:
 542         if(close_sock) {
 543                 close_connection(sock, CONNECTION_CLOSE_PROPERLY);
 544         }
 545 err:
 546         return retval;
 547 }
 548
 549 /*
 550  * fill 512 byte buffer 'buf' with a hashed selection of interesting data based
 551  * only on handle and blknum. The first word is blknum, and the second handle, for ease
 552  * of understanding. Things with handle 0 are blank.
 553  */
 554 static inline void makebuf(char *buf, uint64_t seq, uint64_t blknum) {
 555         uint64_t x = ((uint64_t)blknum) ^ (seq << 32) ^ (seq >> 32);
 556         uint64_t* p = (uint64_t*)buf;
 557         int i;
 558         if (!seq) {
 559                 bzero(buf, 512);
 560                 return;
 561         }
 562         for (i = 0; i<512/sizeof(uint64_t); i++) {
 563                 int s;
 564                 *(p++) = x;
 565                 x+=0xFEEDA1ECDEADBEEFULL+i+(((uint64_t)i)<<56);
 566                 s = x & 63;
 567                 x = x ^ (x<<s) ^ (x>>(64-s)) ^ 0xAA55AA55AA55AA55ULL ^ seq;
 568         }
 569 }
 570
 571 static inline int checkbuf(char *buf, uint64_t seq, uint64_t blknum) {
 572         char cmp[512];
 573         makebuf(cmp, seq, blknum);
 574         return memcmp(cmp, buf, 512)?-1:0;
 575 }
 576
 577 static inline void dumpcommand(char * text, uint32_t command)
 578 {
 579 #ifdef DEBUG_COMMANDS
 580         command=ntohl(command);
 581         char * ctext;
 582         switch (command & NBD_CMD_MASK_COMMAND) {
 583         case NBD_CMD_READ:
 584                 ctext="NBD_CMD_READ";
 585                 break;
 586         case NBD_CMD_WRITE:
 587                 ctext="NBD_CMD_WRITE";
 588                 break;
 589         case NBD_CMD_DISC:
 590                 ctext="NBD_CMD_DISC";
 591                 break;
 592         case NBD_CMD_FLUSH:
 593                 ctext="NBD_CMD_FLUSH";
 594                 break;
 595         default:
 596                 ctext="UNKNOWN";
 597                 break;
 598         }
 599         printf("%s: %s [%s] (0x%08x)\n",
 600                text,
 601                ctext,
 602                (command & NBD_CMD_FLAG_FUA)?"FUA":"NONE",
 603                command);
 604 #endif
 605 }
 606
 607 int integrity_test(gchar* hostname, int port, char* name, int sock,
 608                    char sock_is_open, char close_sock, int testflags) {
 609         struct nbd_reply rep;
 610         fd_set rset;
 611         fd_set wset;
 612         struct timeval tv;
 613         struct timeval start;
 614         struct timeval stop;
 615         double timespan;
 616         double speed;
 617         char speedchar[2] = { '\0', '\0' };
 618         int retval=0;
 619         int serverflags = 0;
 620         pid_t G_GNUC_UNUSED mypid = getpid();
 621         int blkhashfd = -1;
 622         char *blkhashname=NULL;
 623         uint32_t *blkhash = NULL;
 624         int logfd=-1;
 625         uint64_t seq=1;
 626         uint64_t processed=0;
 627         uint64_t printer=0;
 628         int readtransactionfile = 1;
 629         struct rclist txqueue={NULL, NULL, 0};
 630         struct rclist inflight={NULL, NULL, 0};
 631
 632         size=0;
 633         if(!sock_is_open) {
 634                 if((sock=setup_connection(hostname, port, name, CONNECTION_TYPE_FULL, &serverflags))<0) {
 635                         g_warning("Could not open socket: %s", errstr);
 636                         retval=-1;
 637                         goto err;
 638                 }
 639         }
 640
 641         if ((serverflags & (NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA))
 642             != (NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA))
 643                 g_warning("Server flags do not support FLUSH and FUA - these may error");
 644
 645 #ifdef HAVE_MKSTEMP
 646         blkhashname=strdup("/tmp/blkarray-XXXXXX");
 647         if (!blkhashname || (-1 == (blkhashfd = mkstemp(blkhashname)))) {
 648                 g_warning("Could not open temp file: %s", strerror(errno));
 649                 retval=-1;
 650                 goto err;
 651         }
 652 #else
 653         /* use tmpnam here to avoid further feature test nightmare */
 654         if (-1 == (blkhashfd = open(blkhashname=strdup(tmpnam(NULL)),
 655                                     O_CREAT | O_RDWR,
 656                                     S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH))) {
 657                 g_warning("Could not open temp file: %s", strerror(errno));
 658                 retval=-1;
 659                 goto err;
 660         }
 661 #endif
 662         /* Ensure space freed if we die */
 663         if (-1 == unlink(blkhashname)) {
 664                 g_warning("Could not unlink temp file: %s", strerror(errno));
 665                 retval=-1;
 666                 goto err;
 667         }
 668
 669         if (-1 == lseek(blkhashfd, (off_t)((size>>9)<<2), SEEK_SET)) {
 670                 g_warning("Could not llseek temp file: %s", strerror(errno));
 671                 retval=-1;
 672                 goto err;
 673         }
 674
 675         if (-1 == write(blkhashfd, "\0", 1)) {
 676                 g_warning("Could not write temp file: %s", strerror(errno));
 677                 retval=-1;
 678                 goto err;
 679         }
 680
 681         if (NULL == (blkhash = mmap(NULL,
 682                                     (size>>9)<<2,
 683                                     PROT_READ | PROT_WRITE,
 684                                     MAP_SHARED,
 685                                     blkhashfd,
 686                                     0))) {
 687                 g_warning("Could not mmap temp file: %s", strerror(errno));
 688                 retval=-1;
 689                 goto err;
 690         }
 691
 692         if (-1 == (logfd = open(transactionlog, O_RDONLY)))
 693         {
 694                 g_warning("Could open log file: %s", strerror(errno));
 695                 retval=-1;
 696                 goto err;
 697         }
 698
 699         if(gettimeofday(&start, NULL)<0) {
 700                 retval=-1;
 701                 snprintf(errstr, errstr_len, "Could not measure start time: %s", strerror(errno));
 702                 goto err_open;
 703         }
 704
 705         while (readtransactionfile || txqueue.numitems || inflight.numitems) {
 706                 int ret;
 707
 708                 uint32_t magic;
 709                 uint32_t command;
 710                 uint64_t from;
 711                 uint32_t len;
 712                 struct reqcontext * prc;
 713
 714                 *errstr=0;
 715
 716                 FD_ZERO(&wset);
 717                 FD_ZERO(&rset);
 718                 if (readtransactionfile)
 719                         FD_SET(logfd, &rset);
 720                 if (txqueue.numitems)
 721                         FD_SET(sock, &wset);
 722                 if (inflight.numitems)
 723                         FD_SET(sock, &rset);
 724                 tv.tv_sec=5;
 725                 tv.tv_usec=0;
 726                 ret = select(1+((sock>logfd)?sock:logfd), &rset, &wset, NULL, &tv);
 727                 if (ret == 0) {
 728                         retval=-1;
 729                         snprintf(errstr, errstr_len, "Timeout reading from socket");
 730                         goto err_open;
 731                 } else if (ret<0) {
 732                         g_warning("Could not mmap temp file: %s", errstr);
 733                         retval=-1;
 734                         goto err;
 735                 }
 736                 /* We know we've got at least one thing to do here then */
 737
 738                 /* Get a command from the transaction log */
 739                 if (FD_ISSET(logfd, &rset)) {
 740
 741                         /* Read a request or reply from the transaction file */
 742                         READ_ALL_ERRCHK(logfd,
 743                                         &magic,
 744                                         sizeof(magic),
 745                                         err_open,
 746                                         "Could not read transaction log: %s",
 747                                         strerror(errno));
 748                         magic = ntohl(magic);
 749                         switch (magic) {
 750                         case NBD_REQUEST_MAGIC:
 751                                 if (NULL == (prc = calloc(1, sizeof(struct reqcontext)))) {
 752                                         retval=-1;
 753                                         snprintf(errstr, errstr_len, "Could not allocate request");
 754                                         goto err_open;
 755                                 }
 756                                 READ_ALL_ERRCHK(logfd,
 757                                                 sizeof(magic)+(char *)&(prc->req),
 758                                                 sizeof(struct nbd_request)-sizeof(magic),
 759                                                 err_open,
 760                                                 "Could not read transaction log: %s",
 761                                                 strerror(errno));
 762                                 prc->req.magic = htonl(NBD_REQUEST_MAGIC);
 763                                 prc->seq=seq++;
 764                                 if ((ntohl(prc->req.type) & NBD_CMD_MASK_COMMAND) == NBD_CMD_DISC) {
 765                                         /* no more to read; don't enqueue as no reply
 766                                          * we will disconnect manually at the end
 767                                          */
 768                                         readtransactionfile = 0;
 769                                         free (prc);
 770                                 } else {
 771                                         dumpcommand("Enqueuing command", prc->req.type);
 772                                         rclist_addtail(&txqueue, prc);
 773                                 }
 774                                 prc = NULL;
 775                                 break;
 776                         case NBD_REPLY_MAGIC:
 777                                 READ_ALL_ERRCHK(logfd,
 778                                                 sizeof(magic)+(char *)(&rep),
 779                                                 sizeof(struct nbd_reply)-sizeof(magic),
 780                                                 err_open,
 781                                                 "Could not read transaction log: %s",
 782                                                 strerror(errno));
 783
 784                                 if (rep.error) {
 785                                         retval=-1;
 786                                         snprintf(errstr, errstr_len, "Transaction log file contained errored transaction");
 787                                         goto err_open;
 788                                 }
 789
 790                                 /* We do not need to consume data on a read reply as there is
 791                                  * none in the log */
 792                                 break;
 793                         default:
 794                                 retval=-1;
 795                                 snprintf(errstr, errstr_len, "Could not measure start time: %08x", magic);
 796                                 goto err_open;
 797                         }
 798                 }
 799
 800                 /* See if we have a write we can do */
 801                 if (FD_ISSET(sock, &wset))
 802                 {
 803                         prc = txqueue.head;
 804                         if (!prc)
 805                                 g_warning("Socket write FD set but we shouldn't have been interested");
 806                         else
 807                         {
 808
 809                                 rclist_unlink(&txqueue, prc);
 810                                 rclist_addtail(&inflight, prc);
 811
 812                                 if (ntohl(prc->req.magic) != NBD_REQUEST_MAGIC) {
 813                                         retval=-1;
 814                                         g_warning("Asked to write a reply without a magic number");
 815                                         goto err_open;
 816                                 }
 817
 818                                 dumpcommand("Sending command", prc->req.type);
 819                                 command = ntohl(prc->req.type);
 820                                 from = ntohll(prc->req.from);
 821                                 len = ntohl(prc->req.len);
 822                                 /* we rewrite the handle as they otherwise may not be unique */
 823                                 *((uint64_t*)(prc->req.handle))=htonll((uint64_t)prc);
 824                                 WRITE_ALL_ERRCHK(sock,
 825                                                  &(prc->req),
 826                                                  sizeof(struct nbd_request),
 827                                                  err_open,
 828                                                  "Could not write command: %s",
 829                                                  strerror(errno));
 830                                 switch (command & NBD_CMD_MASK_COMMAND) {
 831                                 case NBD_CMD_WRITE:
 832                                         while (len > 0) {
 833                                                 uint64_t blknum = from>>9;
 834                                                 char dbuf[512];
 835                                                 if (from>=size) {
 836                                                         snprintf(errstr, errstr_len, "offset %llx beyond size %llx",
 837                                                                  (long long int) from, (long long int)size);
 838                                                         goto err_open;
 839                                                 }
 840                                                 /* work out what we should be writing */
 841                                                 makebuf(dbuf, prc->seq, blknum);
 842                                                 WRITE_ALL_ERRCHK(sock,
 843                                                                  dbuf,
 844                                                                  512,
 845                                                                  err_open,
 846                                                                  "Could not write data: %s",
 847                                                                  strerror(errno));
 848                                                 from += 512;
 849                                                 len -= 512;
 850                                         }
 851
 852                                 case NBD_CMD_DISC:
 853                                 case NBD_CMD_READ:
 854                                 case NBD_CMD_FLUSH:
 855                                         break;
 856                                 default:
 857                                         retval=-1;
 858                                         snprintf(errstr, errstr_len, "Incomprehensible command: %08x", command);
 859                                         goto err_open;
 860                                         break;
 861                                 }
 862
 863                                 prc = NULL;
 864                         }
 865
 866                 }
 867
 868                 /* See if there is a reply to be processed from the socket */
 869                 if(FD_ISSET(sock, &rset)) {
 870                         /* Okay, there's something ready for
 871                          * reading here */
 872
 873                         READ_ALL_ERRCHK(sock,
 874                                         &rep,
 875                                         sizeof(struct nbd_reply),
 876                                         err_open,
 877                                         "Could not read from server socket: %s",
 878                                         strerror(errno));
 879
 880                         if (rep.magic != htonl(NBD_REPLY_MAGIC)) {
 881                                 retval=-1;
 882                                 snprintf(errstr, errstr_len, "Bad magic from server");
 883                                 goto err_open;
 884                         }
 885
 886                         if (rep.error) {
 887                                 retval=-1;
 888                                 snprintf(errstr, errstr_len, "Server errored a transaction");
 889                                 goto err_open;
 890                         }
 891
 892                         prc=(struct reqcontext *)ntohll(*((uint64_t *)rep.handle));
 893                         if (prc->req.magic != htonl(NBD_REQUEST_MAGIC)) {
 894                                 retval=-1;
 895                                 snprintf(errstr, errstr_len, "Bad magic in inflight data: %08x", prc->req.magic);
 896                                 goto err_open;
 897                         }
 898
 899                         dumpcommand("Processing reply to command", prc->req.type);
 900                         command = ntohl(prc->req.type);
 901                         from = ntohll(prc->req.from);
 902                         len = ntohl(prc->req.len);
 903
 904                         switch (command & NBD_CMD_MASK_COMMAND) {
 905                         case NBD_CMD_READ:
 906                                 while (len > 0) {
 907                                         uint64_t blknum = from>>9;
 908                                         char dbuf[512];
 909                                         if (from>=size) {
 910                                                 snprintf(errstr, errstr_len, "offset %llx beyond size %llx",
 911                                                          (long long int) from, (long long int)size);
 912                                                 goto err_open;
 913                                         }
 914                                         READ_ALL_ERRCHK(sock,
 915                                                         dbuf,
 916                                                         512,
 917                                                         err_open,
 918                                                         "Could not read data: %s",
 919                                                         strerror(errno));
 920                                         /* work out what we was written */
 921                                         if (checkbuf(dbuf, blkhash[blknum], blknum))
 922                                         {
 923                                                 retval=-1;
 924                                                 snprintf(errstr, errstr_len, "Bad reply data: seq %08x", blkhash[blknum]);
 925                                                 goto err_open;
 926
 927                                         }
 928                                         from += 512;
 929                                         len -= 512;
 930                                 }
 931                                 break;
 932                         case NBD_CMD_WRITE:
 933                                 /* subsequent reads should get data with this seq*/
 934                                 while (len > 0) {
 935                                         uint64_t blknum = from>>9;
 936                                         blkhash[blknum]=(uint32_t)(prc->seq);
 937                                         from += 512;
 938                                         len -= 512;
 939                                 }
 940                                 break;
 941                         default:
 942                                 break;
 943                         }
 944
 945                         processed++;
 946                         rclist_unlink(&inflight, prc);
 947                         prc->req.magic=0; /* so a duplicate reply is detected */
 948                         free(prc);
 949                 }
 950
 951                 if (!(printer++ % 10000) || !(readtransactionfile || txqueue.numitems || inflight.numitems) )
 952                         printf("%d: Seq %08lld Queued: %08d Inflight: %08d Done: %08lld\n",
 953                                (int)mypid,
 954                                (long long int) seq,
 955                                txqueue.numitems,
 956                                inflight.numitems,
 957                                (long long int) processed);
 958
 959         }
 960
 961         if (gettimeofday(&stop, NULL)<0) {
 962                 retval=-1;
 963                 snprintf(errstr, errstr_len, "Could not measure end time: %s", strerror(errno));
 964                 goto err_open;
 965         }
 966         timespan=timeval_diff_to_double(&stop, &start);
 967         speed=size/timespan;
 968         if(speed>1024) {
 969                 speed=speed/1024.0;
 970                 speedchar[0]='K';
 971         }
 972         if(speed>1024) {
 973                 speed=speed/1024.0;
 974                 speedchar[0]='M';
 975         }
 976         if(speed>1024) {
 977                 speed=speed/1024.0;
 978                 speedchar[0]='G';
 979         }
 980         g_message("%d: Integrity %s test complete. Took %.3f seconds to complete, %.3f%sib/s", (int)getpid(), (testflags & TEST_WRITE)?"write":"read", timespan, speed, speedchar);
 981
 982 err_open:
 983         if(close_sock) {
 984                 close_connection(sock, CONNECTION_CLOSE_PROPERLY);
 985         }
 986 err:
 987         if (size && blkhash)
 988                 munmap(blkhash, (size>>9)<<2);
 989
 990         if (blkhashfd != -1)
 991                 close (blkhashfd);
 992
 993         if (logfd != -1)
 994                 close (logfd);
 995
 996         if (blkhashname)
 997                 free(blkhashname);
 998
 999         if (*errstr)
1000                 g_warning("%s",errstr);
1001
1002         return retval;
1003 }
1004
1005 typedef int (*testfunc)(gchar*, int, char*, int, char, char, int);
1006
1007 int main(int argc, char**argv) {
1008         gchar *hostname;
1009         long int p = 0;
1010         char* name = NULL;
1011         int sock=0;
1012         int c;
1013         bool want_port = TRUE;
1014         int nonopt=0;
1015         int testflags=0;
1016         testfunc test = throughput_test;
1017
1018         if(argc<3) {
1019                 g_message("%d: Not enough arguments", (int)getpid());
1020                 g_message("%d: Usage: %s <hostname> <port>", (int)getpid(), argv[0]);
1021                 g_message("%d: Or: %s <hostname> -N <exportname>", (int)getpid(), argv[0]);
1022                 exit(EXIT_FAILURE);
1023         }
1024         logging();
1025         while((c=getopt(argc, argv, "-N:t:owfi"))>=0) {
1026                 switch(c) {
1027                         case 1:
1028                                 switch(nonopt) {
1029                                         case 0:
1030                                                 hostname=g_strdup(optarg);
1031                                                 nonopt++;
1032                                                 break;
1033                                         case 1:
1034                                                 if(want_port)
1035                                                 p=(strtol(argv[2], NULL, 0));
1036                                                 if(p==LONG_MIN||p==LONG_MAX) {
1037                                                         g_critical("Could not parse port number: %s", strerror(errno));
1038                                                         exit(EXIT_FAILURE);
1039                                                 }
1040                                                 break;
1041                                 }
1042                                 break;
1043                         case 'N':
1044                                 name=g_strdup(optarg);
1045                                 p = 10809;
1046                                 want_port = false;
1047                                 break;
1048                         case 't':
1049                                 transactionlog=g_strdup(optarg);
1050                                 break;
1051                         case 'o':
1052                                 test=oversize_test;
1053                                 break;
1054                         case 'w':
1055                                 testflags|=TEST_WRITE;
1056                                 break;
1057                         case 'f':
1058                                 testflags|=TEST_FLUSH;
1059                                 break;
1060                         case 'i':
1061                                 test=integrity_test;
1062                                 break;
1063                 }
1064         }
1065
1066         if(test(hostname, (int)p, name, sock, FALSE, TRUE, testflags)<0) {
1067                 g_warning("Could not run test: %s", errstr);
1068                 exit(EXIT_FAILURE);
1069         }
1070
1071         return 0;
1072 }