Add ability to monitor both ends of secchan connections from dpctl.

diff --git a/include/rconn.h b/include/rconn.h
index df17a5c013e6804dac8c97ca3ecf737545f853c0..223e43266dd5462003b905ca7de202708bb15be1 100644 (file)
--- a/include/rconn.h
+++ b/include/rconn.h
@@ -73,6 +73,8 @@ int rconn_send_with_limit(struct rconn *, struct buffer *,
 unsigned int rconn_packets_sent(const struct rconn *);
 unsigned int rconn_packets_received(const struct rconn *);
 
 unsigned int rconn_packets_sent(const struct rconn *);
 unsigned int rconn_packets_received(const struct rconn *);
 

+void rconn_add_monitor(struct rconn *, struct vconn *);
+

 const char *rconn_get_name(const struct rconn *);
 bool rconn_is_alive(const struct rconn *);
 bool rconn_is_connected(const struct rconn *);
 const char *rconn_get_name(const struct rconn *);
 bool rconn_is_alive(const struct rconn *);
 bool rconn_is_connected(const struct rconn *);

diff --git a/lib/rconn.c b/lib/rconn.c
index c1c2748c3550d3ca1f2615f75bc91dfe41d6d259..991e386e3201622543a711f9c38fa43d2a679373 100644 (file)
--- a/lib/rconn.c
+++ b/lib/rconn.c
@@ -114,6 +114,11 @@ struct rconn {
      * an echo request as an inactivity probe packet.  We should receive back
      * a response. */
     int probe_interval;         /* Secs of inactivity before sending probe. */
      * an echo request as an inactivity probe packet.  We should receive back
      * a response. */
     int probe_interval;         /* Secs of inactivity before sending probe. */

+
+    /* Messages sent or received are copied to the monitor connections. */
+#define MAX_MONITORS 8
+    struct vconn *monitors[8];
+    size_t n_monitors;

 };
 
 static unsigned int elapsed_in_this_state(const struct rconn *);
 };
 
 static unsigned int elapsed_in_this_state(const struct rconn *);


@@ -125,6 +130,7 @@ static int reconnect(struct rconn *);
 static void disconnect(struct rconn *, int error);
 static void flush_queue(struct rconn *);
 static void question_connectivity(struct rconn *);
 static void disconnect(struct rconn *, int error);
 static void flush_queue(struct rconn *);
 static void question_connectivity(struct rconn *);

+static void copy_to_monitor(struct rconn *, const struct buffer *);

 
 /* Creates a new rconn, connects it (reliably) to 'name', and returns it. */
 struct rconn *
 
 /* Creates a new rconn, connects it (reliably) to 'name', and returns it. */
 struct rconn *


@@ -189,6 +195,8 @@ rconn_create(int probe_interval, int max_backoff)
 
     rc->probe_interval = probe_interval ? MAX(5, probe_interval) : 0;
 
 
     rc->probe_interval = probe_interval ? MAX(5, probe_interval) : 0;
 

+    rc->n_monitors = 0;
+

     return rc;
 }
 
     return rc;
 }
 


@@ -429,6 +437,7 @@ rconn_recv(struct rconn *rc)
         struct buffer *buffer;
         int error = vconn_recv(rc->vconn, &buffer);
         if (!error) {
         struct buffer *buffer;
         int error = vconn_recv(rc->vconn, &buffer);
         if (!error) {

+            copy_to_monitor(rc, buffer);

             rc->last_received = time_now();
             rc->packets_received++;
             if (rc->state == S_IDLE) {
             rc->last_received = time_now();
             rc->packets_received++;
             if (rc->state == S_IDLE) {


@@ -469,6 +478,7 @@ int
 rconn_send(struct rconn *rc, struct buffer *b, int *n_queued)
 {
     if (rconn_is_connected(rc)) {
 rconn_send(struct rconn *rc, struct buffer *b, int *n_queued)
 {
     if (rconn_is_connected(rc)) {

+        copy_to_monitor(rc, b);

         b->private = n_queued;
         if (n_queued) {
             ++*n_queued;
         b->private = n_queued;
         if (n_queued) {
             ++*n_queued;


@@ -516,6 +526,21 @@ rconn_packets_sent(const struct rconn *rc)
     return rc->packets_sent;
 }
 
     return rc->packets_sent;
 }
 

+/* Adds 'vconn' to 'rc' as a monitoring connection, to which all messages sent
+ * and received on 'rconn' will be copied.  'rc' takes ownership of 'vconn'. */
+void
+rconn_add_monitor(struct rconn *rc, struct vconn *vconn)
+{
+    if (rc->n_monitors < ARRAY_SIZE(rc->monitors)) {
+        VLOG_WARN("new monitor connection from %s", vconn_get_name(vconn));
+        rc->monitors[rc->n_monitors++] = vconn;
+    } else {
+        VLOG_DBG("too many monitor connections, discarding %s",
+                 vconn_get_name(vconn));
+        vconn_close(vconn);
+    }
+}
+

 /* Returns 'rc''s name (the 'name' argument passed to rconn_new()). */
 const char *
 rconn_get_name(const struct rconn *rc)
 /* Returns 'rc''s name (the 'name' argument passed to rconn_new()). */
 const char *
 rconn_get_name(const struct rconn *rc)


@@ -767,3 +792,31 @@ question_connectivity(struct rconn *rc)
         rc->last_questioned = now;
     }
 }
         rc->last_questioned = now;
     }
 }

+
+static void
+copy_to_monitor(struct rconn *rc, const struct buffer *b)
+{
+    struct buffer *clone = NULL;
+    int retval;
+    size_t i;
+
+    for (i = 0; i < rc->n_monitors; ) {
+        struct vconn *vconn = rc->monitors[i];
+
+        if (!clone) {
+            clone = buffer_clone(b);
+        }
+        retval = vconn_send(vconn, clone);
+        if (!retval) {
+            clone = NULL;
+        } else if (retval != EAGAIN) {
+            VLOG_DBG("%s: closing monitor connection to %s: %s",
+                     rconn_get_name(rc), vconn_get_name(vconn),
+                     strerror(retval));
+            rc->monitors[i] = rc->monitors[--rc->n_monitors];
+            continue;
+        }
+        i++;
+    }
+    buffer_delete(clone);
+}

diff --git a/secchan/secchan.8.in b/secchan/secchan.8.in
index e3d297e11d4b525a00df750ffb48820e54939784..c04e8d31131b3edf397eb4bb017bfd9b990bc181 100644 (file)
--- a/secchan/secchan.8.in
+++ b/secchan/secchan.8.in
@@ -295,11 +295,35 @@ are mandatory when this form is used.
 .TP
 \fBptcp:\fR[\fIport\fR]
 Listens for TCP connections on \fIport\fR (default: 975).
 .TP
 \fBptcp:\fR[\fIport\fR]
 Listens for TCP connections on \fIport\fR (default: 975).

-.RE

 
 .TP
 \fBpunix:\fIfile\fR
 Listens for connections on Unix domain server socket named \fIfile\fR.
 
 .TP
 \fBpunix:\fIfile\fR
 Listens for connections on Unix domain server socket named \fIfile\fR.

+.RE
+
+.TP
+\fB-m\fR, \fB--monitor=\fImethod\fR
+Configures the switch to additionally listen for incoming OpenFlow
+connections for switch monitoring with \fBdpctl\fR's \fBmonitor\fR
+command.  The \fImethod\fR must be given as one of the passive
+OpenFlow connection methods listed above as acceptable for
+\fB--listen\fR.
+
+When \fBdpctl monitor\fR makes a monitoring connection, \fBsecchan\fR
+sends it a copy of every OpenFlow message sent to or received from the
+kernel in the normal course of its operations.  It does not send a
+copy of any messages sent to or from the OpenFlow connection to the
+controller.  Most of these messages will be seen anyhow, however,
+because \fBsecchan\fR mainly acts as a relay between the controller
+and the kernel.  \fBsecchan\fR also does not send a copy of any
+messages sent to or from the OpenFlow connection to the controller.
+Such messages will typically \fBnot\fR be seen, because \fBsecchan\fR
+maintains a separate connection to the kernel for each management
+connection.
+
+Messages are copied to the monitoring connections on a best-effort
+basis.  In particular, if the socket buffer of the monitoring
+connection fills up, some messages will be lost.

 
 .TP
 \fB-p\fR, \fB--private-key=\fIprivkey.pem\fR
 
 .TP
 \fB-p\fR, \fB--private-key=\fIprivkey.pem\fR

diff --git a/secchan/secchan.c b/secchan/secchan.c
index bd2d1ed74b2c1800d7b11398513a800b8f70cc87..5a41cc2043db4896819037292f4e7ad99e88cfab 100644 (file)
--- a/secchan/secchan.c
+++ b/secchan/secchan.c
@@ -92,6 +92,7 @@ struct settings {
     const char *controller_name; /* Controller (if not discovery mode). */
     const char *listener_names[MAX_MGMT]; /* Listen for mgmt connections. */
     size_t n_listeners;          /* Number of mgmt connection listeners. */
     const char *controller_name; /* Controller (if not discovery mode). */
     const char *listener_names[MAX_MGMT]; /* Listen for mgmt connections. */
     size_t n_listeners;          /* Number of mgmt connection listeners. */

+    const char *monitor_name;   /* Listen for traffic monitor connections. */

 
     /* Failure behavior. */
     enum fail_mode fail_mode; /* Act as learning switch if no controller? */
 
     /* Failure behavior. */
     enum fail_mode fail_mode; /* Act as learning switch if no controller? */


@@ -137,6 +138,9 @@ static struct vlog_rate_limit vrl = VLOG_RATE_LIMIT_INIT(60, 60);
 static void parse_options(int argc, char *argv[], struct settings *);
 static void usage(void) NO_RETURN;
 
 static void parse_options(int argc, char *argv[], struct settings *);
 static void usage(void) NO_RETURN;
 

+static struct vconn *open_passive_vconn(const char *name);
+static struct vconn *accept_vconn(struct vconn *vconn);
+

 static struct relay *relay_create(struct rconn *local, struct rconn *remote,
                                   bool is_mgmt_conn);
 static struct relay *relay_accept(const struct settings *, struct vconn *);
 static struct relay *relay_create(struct rconn *local, struct rconn *remote,
                                   bool is_mgmt_conn);
 static struct relay *relay_accept(const struct settings *, struct vconn *);


@@ -195,6 +199,8 @@ main(int argc, char *argv[])
     struct hook hooks[8];
     size_t n_hooks = 0;
 
     struct hook hooks[8];
     size_t n_hooks = 0;
 

+    struct vconn *monitor;
+

     struct vconn *listeners[MAX_MGMT];
     size_t n_listeners;
 
     struct vconn *listeners[MAX_MGMT];
     size_t n_listeners;
 


@@ -212,20 +218,12 @@ main(int argc, char *argv[])
     parse_options(argc, argv, &s);
     signal(SIGPIPE, SIG_IGN);
 
     parse_options(argc, argv, &s);
     signal(SIGPIPE, SIG_IGN);
 

-    /* Start listening for management connections. */
+    /* Start listening for management and monitoring connections. */

     n_listeners = 0;
     for (i = 0; i < s.n_listeners; i++) {
     n_listeners = 0;
     for (i = 0; i < s.n_listeners; i++) {

-        const char *name = s.listener_names[i];
-        struct vconn *listener;
-        retval = vconn_open(name, &listener);
-        if (retval && retval != EAGAIN) {
-            fatal(retval, "opening %s", name);
-        }
-        if (!vconn_is_passive(listener)) {
-            fatal(0, "%s is not a passive vconn", name);
-        }
-        listeners[n_listeners++] = listener;
+        listeners[n_listeners++] = open_passive_vconn(s.listener_names[i]);

     }
     }

+    monitor = s.monitor_name ? open_passive_vconn(s.monitor_name) : NULL;

 
     /* Initialize switch status hook. */
     hooks[n_hooks++] = switch_status_hook_create(&s, &switch_status);
 
     /* Initialize switch status hook. */
     hooks[n_hooks++] = switch_status_hook_create(&s, &switch_status);


@@ -298,6 +296,12 @@ main(int argc, char *argv[])
                 list_push_back(&relays, &r->node);
             }
         }
                 list_push_back(&relays, &r->node);
             }
         }

+        if (monitor) {
+            struct vconn *new = accept_vconn(monitor);
+            if (new) {
+                rconn_add_monitor(local_rconn, new);
+            }
+        }

         for (i = 0; i < n_hooks; i++) {
             if (hooks[i].periodic_cb) {
                 hooks[i].periodic_cb(hooks[i].aux);
         for (i = 0; i < n_hooks; i++) {
             if (hooks[i].periodic_cb) {
                 hooks[i].periodic_cb(hooks[i].aux);


@@ -324,6 +328,9 @@ main(int argc, char *argv[])
         for (i = 0; i < n_listeners; i++) {
             vconn_accept_wait(listeners[i]);
         }
         for (i = 0; i < n_listeners; i++) {
             vconn_accept_wait(listeners[i]);
         }

+        if (monitor) {
+            vconn_accept_wait(monitor);
+        }

         for (i = 0; i < n_hooks; i++) {
             if (hooks[i].wait_cb) {
                 hooks[i].wait_cb(hooks[i].aux);
         for (i = 0; i < n_hooks; i++) {
             if (hooks[i].wait_cb) {
                 hooks[i].wait_cb(hooks[i].aux);


@@ -338,6 +345,35 @@ main(int argc, char *argv[])
     return 0;
 }
 
     return 0;
 }
 

+static struct vconn *
+open_passive_vconn(const char *name) 
+{
+    struct vconn *vconn;
+    int retval;
+
+    retval = vconn_open(name, &vconn);
+    if (retval && retval != EAGAIN) {
+        fatal(retval, "opening %s", name);
+    }
+    if (!vconn_is_passive(vconn)) {
+        fatal(0, "%s is not a passive vconn", name);
+    }
+    return vconn;
+}
+
+static struct vconn *
+accept_vconn(struct vconn *vconn) 
+{
+    struct vconn *new;
+    int retval;
+
+    retval = vconn_accept(vconn, &new);
+    if (retval && retval != EAGAIN) {
+        VLOG_WARN_RL(&vrl, "accept failed (%s)", strerror(retval));
+    }
+    return new;
+}
+

 static struct hook
 make_hook(bool (*packet_cb)(struct relay *, int half, void *aux),
           void (*periodic_cb)(void *aux),
 static struct hook
 make_hook(bool (*packet_cb)(struct relay *, int half, void *aux),
           void (*periodic_cb)(void *aux),


@@ -362,11 +398,8 @@ relay_accept(const struct settings *s, struct vconn *listen_vconn)
     struct rconn *r1, *r2;
     int retval;
 
     struct rconn *r1, *r2;
     int retval;
 

-    retval = vconn_accept(listen_vconn, &new_remote);
-    if (retval) {
-        if (retval != EAGAIN) {
-            VLOG_WARN_RL(&vrl, "accept failed (%s)", strerror(retval));
-        }
+    new_remote = accept_vconn(listen_vconn);
+    if (!new_remote) {

         return NULL;
     }
 
         return NULL;
     }
 


@@ -1374,6 +1407,7 @@ parse_options(int argc, char *argv[], struct settings *s)
         {"max-idle",    required_argument, 0, OPT_MAX_IDLE},
         {"max-backoff", required_argument, 0, OPT_MAX_BACKOFF},
         {"listen",      required_argument, 0, 'l'},
         {"max-idle",    required_argument, 0, OPT_MAX_IDLE},
         {"max-backoff", required_argument, 0, OPT_MAX_BACKOFF},
         {"listen",      required_argument, 0, 'l'},

+        {"monitor",     required_argument, 0, 'm'},

         {"rate-limit",  optional_argument, 0, OPT_RATE_LIMIT},
         {"burst-limit", required_argument, 0, OPT_BURST_LIMIT},
         {"detach",      no_argument, 0, 'D'},
         {"rate-limit",  optional_argument, 0, OPT_RATE_LIMIT},
         {"burst-limit", required_argument, 0, OPT_BURST_LIMIT},
         {"detach",      no_argument, 0, 'D'},


@@ -1391,6 +1425,7 @@ parse_options(int argc, char *argv[], struct settings *s)
 
     /* Set defaults that we can figure out before parsing options. */
     s->n_listeners = 0;
 
     /* Set defaults that we can figure out before parsing options. */
     s->n_listeners = 0;

+    s->monitor_name = NULL;

     s->fail_mode = FAIL_OPEN;
     s->max_idle = 15;
     s->probe_interval = 15;
     s->fail_mode = FAIL_OPEN;
     s->max_idle = 15;
     s->probe_interval = 15;


@@ -1492,6 +1527,13 @@ parse_options(int argc, char *argv[], struct settings *s)
             s->listener_names[s->n_listeners++] = optarg;
             break;
 
             s->listener_names[s->n_listeners++] = optarg;
             break;
 

+        case 'm':
+            if (s->monitor_name) {
+                fatal(0, "-m or --monitor may only be specified once");
+            }
+            s->monitor_name = optarg;
+            break;
+

         case 'h':
             usage();
 
         case 'h':
             usage();
 


@@ -1608,6 +1650,8 @@ usage(void)
            "                          attempts (default: 15 seconds)\n"
            "  -l, --listen=METHOD     allow management connections on METHOD\n"
            "                          (a passive OpenFlow connection method)\n"
            "                          attempts (default: 15 seconds)\n"
            "  -l, --listen=METHOD     allow management connections on METHOD\n"
            "                          (a passive OpenFlow connection method)\n"

+           "  -m, --monitor=METHOD    copy traffic to/from kernel to METHOD\n"
+           "                          (a passive OpenFlow connection method)\n"

            "\nRate-limiting of \"packet-in\" messages to the controller:\n"
            "  --rate-limit[=PACKETS]  max rate, in packets/s (default: 1000)\n"
            "  --burst-limit=BURST     limit on packet credit for idle time\n"
            "\nRate-limiting of \"packet-in\" messages to the controller:\n"
            "  --rate-limit[=PACKETS]  max rate, in packets/s (default: 1000)\n"
            "  --burst-limit=BURST     limit on packet credit for idle time\n"

diff --git a/utilities/dpctl.8 b/utilities/dpctl.8
index 2af2f270a15f37e64471d292b108ebebdfa9cf8c..4c863e5592291e374a67117815954071bd0fdb85 100644 (file)
--- a/utilities/dpctl.8
+++ b/utilities/dpctl.8
@@ -44,7 +44,7 @@ The Unix domain server socket named \fIfile\fR.
 .SH COMMANDS
 
 With the \fBdpctl\fR program, datapaths running in the kernel can be 
 .SH COMMANDS
 
 With the \fBdpctl\fR program, datapaths running in the kernel can be 

-created, deleted, modified, and monitored.  A single machine may 
+created, deleted, and modified.  A single machine may 

 host up to 32 datapaths (numbered 0 to 31).  In most situations, 
 a machine hosts only one datapath.
 
 host up to 32 datapaths (numbered 0 to 31).  In most situations, 
 a machine hosts only one datapath.
 


@@ -83,12 +83,6 @@ traffic and the network device appears silent to the rest of the system.
 Removes each \fInetdev\fR from the list of network devices datapath
 \fIdp_idx\fR monitors.
 
 Removes each \fInetdev\fR from the list of network devices datapath
 \fIdp_idx\fR monitors.
 

-.TP
-\fBmonitor nl:\fIdp_idx\fR
-Prints to the console all OpenFlow packets sent by datapath
-\fIdp_idx\fR to its controller, where \fIdp_idx\fR is the ID of an
-existing datapath.
-

 .PP
 The following commands can be apply to OpenFlow switches regardless of
 the connection method.
 .PP
 The following commands can be apply to OpenFlow switches regardless of
 the connection method.


@@ -182,6 +176,19 @@ Deletes entries from the datapath \fIswitch\fR's tables that match
 tables are removed.  See \fBFLOW SYNTAX\fR, below, for the syntax of
 \fIflows\fR.
 
 tables are removed.  See \fBFLOW SYNTAX\fR, below, for the syntax of
 \fIflows\fR.
 

+.TP
+\fBmonitor \fIswitch\fR
+Connects to \fIswitch\fR and prints to the console all OpenFlow
+messages received.  Usually, \fIswitch\fR should specify a connection
+named on \fBsecchan\fR(8)'s \fB-m\fR or \fB--monitor\fR command line
+option, in which the messages printed will be all those sent or
+received by \fBsecchan\fR to or from the kernel datapath module.  A
+\fIswitch\fR of the form \fBnl:\fIdp_idx\fR will print all
+asynchronously generated OpenFlow messages (such as packet-in
+messages), but it will not print any messages sent to the kernel by
+\fBsecchan\fR and other processes, nor will it print replies sent by
+the kernel in response to those messages.
+

 .PP
 The following commands can be used regardless of the connection
 method.  They apply to OpenFlow switches and controllers.
 .PP
 The following commands can be used regardless of the connection
 method.  They apply to OpenFlow switches and controllers.

diff --git a/utilities/dpctl.c b/utilities/dpctl.c
index 62c93c676430ab523796ee38f297fca809420a9d..a008e3cae89fcf3a1cebd58436500af7edeeef3e 100644 (file)
--- a/utilities/dpctl.c
+++ b/utilities/dpctl.c
@@ -189,7 +189,6 @@ usage(void)
            "  deldp nl:DP_ID              delete local datapath DP_ID\n"
            "  addif nl:DP_ID IFACE...     add each IFACE as a port on DP_ID\n"
            "  delif nl:DP_ID IFACE...     delete each IFACE from DP_ID\n"
            "  deldp nl:DP_ID              delete local datapath DP_ID\n"
            "  addif nl:DP_ID IFACE...     add each IFACE as a port on DP_ID\n"
            "  delif nl:DP_ID IFACE...     delete each IFACE from DP_ID\n"

-           "  monitor nl:DP_ID            print packets received\n"

 #endif
            "\nFor local datapaths and remote switches:\n"
            "  show SWITCH                 show basic information\n"
 #endif
            "\nFor local datapaths and remote switches:\n"
            "  show SWITCH                 show basic information\n"


@@ -205,6 +204,7 @@ usage(void)
            "  add-flow SWITCH FLOW        add flow described by FLOW\n"
            "  add-flows SWITCH FILE       add flows from FILE\n"
            "  del-flows SWITCH FLOW       delete matching FLOWs\n"
            "  add-flow SWITCH FLOW        add flow described by FLOW\n"
            "  add-flows SWITCH FILE       add flows from FILE\n"
            "  del-flows SWITCH FLOW       delete matching FLOWs\n"

+           "  monitor SWITCH              print packets received from SWITCH\n"

            "\nFor local datapaths, remote switches, and controllers:\n"
            "  probe VCONN                 probe whether VCONN is up\n"
            "  ping VCONN [N]              latency of N-byte echos\n"
            "\nFor local datapaths, remote switches, and controllers:\n"
            "  probe VCONN                 probe whether VCONN is up\n"
            "  ping VCONN [N]              latency of N-byte echos\n"


@@ -323,18 +323,6 @@ static void do_del_port(int argc UNUSED, char *argv[])
 {
     add_del_ports(argc, argv, dpif_del_port, "remove", "from");
 }
 {
     add_del_ports(argc, argv, dpif_del_port, "remove", "from");
 }

-
-static void do_monitor(int argc UNUSED, char *argv[])
-{
-    struct dpif dp;
-    open_nl_vconn(argv[1], true, &dp);
-    for (;;) {
-        struct buffer *b;
-        run(dpif_recv_openflow(&dp, &b, true), "dpif_recv_openflow");
-        ofp_print(stderr, b->data, b->size, 2);
-        buffer_delete(b);
-    }
-}

 #endif /* HAVE_NETLINK */
 \f
 /* Generic commands. */
 #endif /* HAVE_NETLINK */
 \f
 /* Generic commands. */


@@ -887,6 +875,29 @@ static void do_del_flows(int argc, char *argv[])
     vconn_close(vconn);
 }
 
     vconn_close(vconn);
 }
 

+static void
+do_monitor(int argc UNUSED, char *argv[])
+{
+    struct vconn *vconn;
+    const char *name;
+
+    /* If the user specified, e.g., "nl:0", append ":1" to it to ensure that
+     * the connection will subscribe to listen for asynchronous messages, such
+     * as packet-in messages. */
+    if (!strncmp(argv[1], "nl:", 3) && strrchr(argv[1], ':') == &argv[1][2]) {
+        name = xasprintf("%s:1", argv[1]);
+    } else {
+        name = argv[1];
+    }
+    run(vconn_open_block(argv[1], &vconn), "connecting to %s", name);
+    for (;;) {
+        struct buffer *b;
+        run(vconn_recv_block(vconn, &b), "vconn_recv");
+        ofp_print(stderr, b->data, b->size, 2);
+        buffer_delete(b);
+    }
+}
+

 static void
 do_dump_ports(int argc, char *argv[])
 {
 static void
 do_dump_ports(int argc, char *argv[])
 {