Automated testing

Makefile

@@ -68,12 +68,14 @@ CFLAGS = $(MD5INCLUDES) $(CDEBUGFLAGS) $(DEFINES) $(EXTRA_DEFINES)
 SRCS = util.c event.c io.c chunk.c atom.c object.c log.c diskcache.c main.c \
        config.c local.c http.c client.c server.c auth.c tunnel.c \
        http_parse.c parse_time.c dns.c forbidden.c \
-       md5import.c md5.c ftsimport.c fts_compat.c socks.c mingw.c
+       md5import.c md5.c ftsimport.c fts_compat.c socks.c mingw.c \
+       sd.c
 
 OBJS = util.o event.o io.o chunk.o atom.o object.o log.o diskcache.o main.o \
        config.o local.o http.o client.o server.o auth.o tunnel.o \
        http_parse.o parse_time.o dns.o forbidden.o \
-       md5import.o ftsimport.o socks.o mingw.o
+       md5import.o ftsimport.o socks.o mingw.o \
+       sd.c
 
 polipo$(EXE): $(OBJS)
 	$(CC) $(CFLAGS) $(LDFLAGS) -o polipo$(EXE) $(OBJS) $(MD5LIBS) $(LDLIBS)
@@ -136,10 +138,20 @@ TAGS: $(SRCS)
 .PHONY: clean
 
 clean:
-	-rm -f polipo$(EXE) *.o *~ core TAGS gmon.out
+	-rm -f polipo$(EXE) *.o *~ core TAGS gmon.out test/http-test-webserver$(EXE) test/sd-launch$(EXE)
 	-rm -f polipo.cp polipo.fn polipo.log polipo.vr
 	-rm -f polipo.cps polipo.info* polipo.pg polipo.toc polipo.vrs
 	-rm -f polipo.aux polipo.dvi polipo.ky polipo.ps polipo.tp
 	-rm -f polipo.dvi polipo.ps polipo.ps.gz polipo.pdf polipo.html
 	-rm -rf ./html/
 	-rm -f polipo.man.html
+
+check: polipo$(EXE) test/http-test-webserver$(EXE) test/sd-launch$(EXE)
+	test/run-test.sh
+
+test/http-test-webserver$(EXE): test/http-test-webserver.c sd.c
+	$(CC) $(CFLAGS) $(LDFLAGS) test/http-test-webserver.c sd.c -o test/http-test-webserver$(EXE) $(MD5LIBS) $(LDLIBS)
+
+test/sd-launch$(EXE): test/sd-launch.c
+	$(CC) $(CFLAGS) $(LDFLAGS) test/sd-launch.c -o test/sd-launch$(EXE) $(MD5LIBS) $(LDLIBS)
+

io.c

@@ -682,14 +682,16 @@ do_scheduled_accept(int status, FdEventHandlerPtr event)
     return done;
 }
 
-FdEventHandlerPtr
-create_listener(char *address, int port,
-                int (*handler)(int, FdEventHandlerPtr, AcceptRequestPtr),
-                void *data)
+/**
+ * Opens a listening socket
+ *
+ * Returns the file descriptor on success, -1 on failure and will set errno.
+ */
+int
+open_listening_socket(char *address, int port)
 {
     int fd, rc;
     int one = 1;
-    int done;
     struct sockaddr_in addr;
 #ifdef HAVE_IPv6
     int inet6 = 1;
@@ -719,9 +721,7 @@ create_listener(char *address, int port,
     }
 
     if(fd < 0) {
-        done = (*handler)(-errno, NULL, NULL);
-        assert(done);
-        return NULL;
+        goto fail;
     }
 
     rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *)&one, sizeof(one));
@@ -738,9 +738,8 @@ create_listener(char *address, int port,
         memset(&addr6, 0, sizeof(addr6));
         rc = inet_pton(AF_INET6, address, &addr6.sin6_addr);
         if(rc != 1) {
-            done = (*handler)(rc == 0 ? -ESYNTAX : -errno, NULL, NULL);
-            assert(done);
-            return NULL;
+            errno = (rc == 0) ? ESYNTAX : errno;
+            goto fail;
         }
         addr6.sin6_family = AF_INET6;
         addr6.sin6_port = htons(port);
@@ -753,9 +752,8 @@ create_listener(char *address, int port,
         memset(&addr, 0, sizeof(addr));
         rc = inet_aton(address, &addr.sin_addr);
         if(rc != 1) {
-            done = (*handler)(rc == 0 ? -ESYNTAX : -errno, NULL, NULL);
-            assert(done);
-            return NULL;
+            errno = (rc == 0) ? ESYNTAX : errno;
+            goto fail;
         }
         addr.sin_family = AF_INET;
         addr.sin_port = htons(port);
@@ -764,33 +762,54 @@ create_listener(char *address, int port,
 
     if(rc < 0) {
         do_log_error(L_ERROR, errno, "Couldn't bind");
-        CLOSE(fd);
-        done = (*handler)(-errno, NULL, NULL);
-        assert(done);
-        return NULL;
+        goto failwithfd;
     }
 
     rc = setNonblocking(fd, 1);
     if(rc < 0) {
         do_log_error(L_ERROR, errno, "Couldn't set non blocking mode");
-        CLOSE(fd);
-        done = (*handler)(-errno, NULL, NULL);
-        assert(done);
-        return NULL;
+        goto failwithfd;
     }
 
     rc = listen(fd, 32);
     if(rc < 0) {
         do_log_error(L_ERROR, errno, "Couldn't listen");
-        CLOSE(fd);
-        done = (*handler)(-errno, NULL, NULL);
-        assert(done);
-        return NULL;
+        goto failwithfd;
     }
 
     do_log(L_INFO, "Established listening socket on port %d.\n", port);
+    return fd;
+failwithfd:
+    CLOSE(fd);
+fail:
+    return -1;
+}
 
-    return schedule_accept(fd, handler, data);
+FdEventHandlerPtr
+create_listener(char *address, int port,
+                int (*handler)(int, FdEventHandlerPtr, AcceptRequestPtr),
+                void *data)
+{
+    int done;
+    int fd;
+
+    /* If the environment variables LISTEN_FDS=1 and LISTEN_PID=getpid() then
+       the socket file descriptor 3 will be used to listen for incoming
+       connections.  If any of this fails we fall back to creating our own
+       socket to listen on.  This is consistant with systemd's socket passing
+       convention. */
+    fd = get_sd_socket();
+    if (fd < 0) {
+        fd = open_listening_socket(address, port);
+    }
+    if (fd < 0) {
+        done = (*handler)(-errno, NULL, NULL);
+        assert(done);
+        return NULL;
+    }
+    else {
+        return schedule_accept(fd, handler, data);
+    }
 }
 
 #ifndef SOL_TCP

polipo.h

@@ -210,6 +210,7 @@ THE SOFTWARE.
 #include "log.h"
 #include "auth.h"
 #include "tunnel.h"
+#include "sd.h"
 
 extern AtomPtr configFile;
 extern int daemonise;

sd.c

@@ -0,0 +1,101 @@
+/*
+Copyright (c) 2011 YouView TV Ltd. <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include "polipo.h"
+#include <errno.h>
+
+int
+get_sd_socket() {
+    int fd = -1;
+    struct stat buf;
+    int fdflags;
+    int type;
+    int mask;
+    const char* env;
+
+    env = getenv("LISTEN_FDS");
+    if (!env || strcmp(env, "1") != 0)
+        goto fail;
+    unsetenv("LISTEN_FDS");
+
+    env = getenv("LISTEN_PID");
+    if (!env || strtoul(env, NULL, 10) != getpid()) {
+        do_log(L_WARN, "Socket passing error: LISTEN_FDS present but "
+               "LISTEN_PID (%s) doesn't match pid (%u)", env, getpid());
+        goto fail;
+    }
+    unsetenv("LISTEN_PID");
+    unsetenv("LISTEN_FDS");
+
+    /* LISTEN_FDS and LISTEN_PID are fine: the socket passing is enabled */
+    fd = 3;
+
+    if (fstat(fd, &buf) != 0) {
+        do_log_error(L_WARN, errno, "Socket passing error: fstating the "
+                     "passed file descriptor failed");
+        goto fail;
+    }
+
+    if (!S_ISSOCK(buf.st_mode)) {
+        do_log(L_WARN, "Socket passing error: File descriptor 3 passed in is "
+               "not a socket");
+        goto fail;
+    }
+
+    socklen_t len = sizeof(type);
+    if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &type, &len) < 0) {
+        do_log_error(L_WARN, errno, "Socket passing error: getsockopt failed");
+        goto fail;
+    }
+
+    if (len != sizeof(type)) {
+        do_log(L_WARN, "Socket passing error: Bizzare: getsockopt failed to "
+               "fill provided buffer");
+        goto fail;
+    }
+
+    if (type != SOCK_STREAM) {
+        do_log_error(L_WARN, errno, "Socket passing error: getsockopt failed");
+        goto fail;
+    }
+
+    mask = fcntl(fd, F_GETFL, 0);
+    if (mask < 0 || fcntl(fd, F_SETFL, mask | O_NONBLOCK) < 0) {
+        do_log_error(L_WARN, errno, "Failed to set socket in non-blocking mode");
+        errno = EBADF;
+        goto fail;
+    }
+
+#ifdef O_CLOEXEC
+    /* It doesn't hurt to set O_CLOEXEC if possible.  Don't really mind if
+       this fails */
+    if ((fdflags = fcntl(fd, F_GETFD)) != -1) {
+        fcntl(fd, F_SETFD, fdflags | O_CLOEXEC);
+    }
+#endif // O_CLOEXEC
+
+    return fd;
+fail:
+    CLOSE(fd);
+    return -1;
+}
+

sd.h

@@ -0,0 +1,28 @@
+/*
+Copyright (c) 2011 YouView TV Ltd. <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ * Returns the file descriptor passed in with systemd style socket passing or
+ * -1 on error.
+ */
+int
+get_sd_socket();

test/.gitattributes

@@ -0,0 +1,6 @@
+# These files must be reproduced verbatim.  HTTP headers use CRLF as a line
+# ending and the length of the payload can be encoded in the header no neither
+# files should be touched by an automatic tool.
+*.header -text
+*.payload -text
+

test/CREDITS

@@ -0,0 +1,16 @@
+expect_cache/kim_jong_il
+    Retrieved from wikipedia 2011-12-19:
+    http://en.wikipedia.org/wiki/File:Kim_Jong-il_on_August_24,_2011.jpg
+    Licence: Creative Commons Attribution 3.0 Unported
+    Original Source: www.kremlin.ru
+
+expect_uncached/egbert
+    Retreived from wikipedia 2011-12-19:
+    http://en.wikipedia.org/wiki/Egbert_of_Wessex
+    Licence: Creative Commons Attribution-ShareAlike License
+
+expect_cached/water_image
+    Retrieved on 2011-12-19
+    Contents removed due to licencing, headers preserved but with modified
+        Content-Length.  Notably Content-Type says image/jpeg which has been
+        intentionally unchanged from the original.

test/README

@@ -0,0 +1,47 @@
+Running the tests
+*****************
+
+In the polipo directory execute:
+
+    $ make check
+
+Creating new tests
+******************
+
+Test cases consist of pairs of .header and .payload files.  These are generated
+by capturing HTTP sessions with curl[1].  If the data should be cached the
+files are placed in the expect_cache directory.  If they shouldn't be cached
+they are placed in the expect_uncached directory.
+
+To generate a test case from an HTTP session use:
+
+    $ curl -o test_name.payload --dump-header test_name.header
+
+[1] http://curl.haxx.se/
+
+How it works
+************
+
+http-test-webserver is a small webserver that listens on a socket and for each
+connection it will send the contents of a file once the client has finished
+sending http headers (signified by \r\n\r\n).  When it terminates it prints
+REQUESTS_SERVED=XXX where XXX is the number of HTTP requests that were made to
+it.  If this is anything other than 1 we know that polipo has not been caching
+the served files.
+
+We use the (systemd) support for providing a listening socket to polipo and
+http-test-webserver so we can have them listen on a different random unused
+port each time so the tests will not interfere with each other and are less
+dependent on the environment within which they are started.  sd-launch creates
+a socket and spins off a process listening on that socket before printing the
+PID of the spun off process and the port of the listening socket.  This is used
+by the run-test.sh script to know what port to pass to curl/set as http_proxy
+and to know how to kill the started processes.
+
+Future directions
+*****************
+
+It would be good to add many more tests, possibly generating headers
+programmatically.  It would also be good to amend http-test-webserver to
+validate that the requests that polipo makes are valid and as expected.
+

test/expect_cache/kim_jong_il.header

@@ -0,0 +1,15 @@
+HTTP/1.0 200 OK
+Content-Length: 15816
+Accept-Ranges: bytes
+Server: Sun-Java-System-Web-Server/7.0
+Date: Thu, 15 Dec 2011 14:35:22 GMT
+Content-Type: image/jpeg
+Last-Modified: Wed, 24 Aug 2011 17:55:10 GMT
+ETag: "3dc8-4e553afe"
+X-Cache: HIT from amssq57.esams.wikimedia.org
+X-Cache-Lookup: HIT from amssq57.esams.wikimedia.org:3128
+Age: 364162
+X-Cache: HIT from amssq56.esams.wikimedia.org
+X-Cache-Lookup: HIT from amssq56.esams.wikimedia.org:80
+Connection: close
+

test/expect_cache/water_image.header

@@ -0,0 +1,8 @@
+HTTP/1.1 200 OK
+ETag: cVdrFtMJAxQiRa2XJMwYjg==
+Content-Type: image/jpeg
+Content-Length: 130
+Cache-Control: max-age=258
+Date: Mon, 12 Dec 2011 20:21:58 GMT
+Connection: keep-alive
+