2525#define MAX_IRQ_MSG_SIZE (sizeof(struct virtio_pcidev_msg) + sizeof(u32))
2626#define NUM_IRQ_MSGS 10
2727
28- #define HANDLE_NO_FREE (ptr ) ((void *)((unsigned long)(ptr) | 1))
29- #define HANDLE_IS_NO_FREE (ptr ) ((unsigned long)(ptr) & 1)
28+ struct um_pci_message_buffer {
29+ struct virtio_pcidev_msg hdr ;
30+ u8 data [8 ];
31+ };
3032
3133struct um_pci_device {
3234 struct virtio_device * vdev ;
@@ -36,6 +38,11 @@ struct um_pci_device {
3638
3739 struct virtqueue * cmd_vq , * irq_vq ;
3840
41+ #define UM_PCI_WRITE_BUFS 20
42+ struct um_pci_message_buffer bufs [UM_PCI_WRITE_BUFS + 1 ];
43+ void * extra_ptrs [UM_PCI_WRITE_BUFS + 1 ];
44+ DECLARE_BITMAP (used_bufs , UM_PCI_WRITE_BUFS );
45+
3946#define UM_PCI_STAT_WAITING 0
4047 unsigned long status ;
4148
@@ -61,12 +68,40 @@ static unsigned long um_pci_msi_used[BITS_TO_LONGS(MAX_MSI_VECTORS)];
6168static unsigned int um_pci_max_delay_us = 40000 ;
6269module_param_named (max_delay_us , um_pci_max_delay_us , uint , 0644 );
6370
64- struct um_pci_message_buffer {
65- struct virtio_pcidev_msg hdr ;
66- u8 data [8 ];
67- };
71+ static int um_pci_get_buf (struct um_pci_device * dev , bool * posted )
72+ {
73+ int i ;
74+
75+ for (i = 0 ; i < UM_PCI_WRITE_BUFS ; i ++ ) {
76+ if (!test_and_set_bit (i , dev -> used_bufs ))
77+ return i ;
78+ }
6879
69- static struct um_pci_message_buffer __percpu * um_pci_msg_bufs ;
80+ * posted = false;
81+ return UM_PCI_WRITE_BUFS ;
82+ }
83+
84+ static void um_pci_free_buf (struct um_pci_device * dev , void * buf )
85+ {
86+ int i ;
87+
88+ if (buf == & dev -> bufs [UM_PCI_WRITE_BUFS ]) {
89+ kfree (dev -> extra_ptrs [UM_PCI_WRITE_BUFS ]);
90+ dev -> extra_ptrs [UM_PCI_WRITE_BUFS ] = NULL ;
91+ return ;
92+ }
93+
94+ for (i = 0 ; i < UM_PCI_WRITE_BUFS ; i ++ ) {
95+ if (buf == & dev -> bufs [i ]) {
96+ kfree (dev -> extra_ptrs [i ]);
97+ dev -> extra_ptrs [i ] = NULL ;
98+ WARN_ON (!test_and_clear_bit (i , dev -> used_bufs ));
99+ return ;
100+ }
101+ }
102+
103+ WARN_ON (1 );
104+ }
70105
71106static int um_pci_send_cmd (struct um_pci_device * dev ,
72107 struct virtio_pcidev_msg * cmd ,
@@ -82,7 +117,9 @@ static int um_pci_send_cmd(struct um_pci_device *dev,
82117 };
83118 struct um_pci_message_buffer * buf ;
84119 int delay_count = 0 ;
120+ bool bounce_out ;
85121 int ret , len ;
122+ int buf_idx ;
86123 bool posted ;
87124
88125 if (WARN_ON (cmd_size < sizeof (* cmd ) || cmd_size > sizeof (* buf )))
@@ -101,66 +138,69 @@ static int um_pci_send_cmd(struct um_pci_device *dev,
101138 break ;
102139 }
103140
104- buf = get_cpu_var (um_pci_msg_bufs );
105- if (buf )
106- memcpy (buf , cmd , cmd_size );
141+ bounce_out = !posted && cmd_size <= sizeof (* cmd ) &&
142+ out && out_size <= sizeof (buf -> data );
107143
108- if (posted ) {
109- u8 * ncmd = kmalloc (cmd_size + extra_size , GFP_ATOMIC );
110-
111- if (ncmd ) {
112- memcpy (ncmd , cmd , cmd_size );
113- if (extra )
114- memcpy (ncmd + cmd_size , extra , extra_size );
115- cmd = (void * )ncmd ;
116- cmd_size += extra_size ;
117- extra = NULL ;
118- extra_size = 0 ;
119- } else {
120- /* try without allocating memory */
121- posted = false;
122- cmd = (void * )buf ;
144+ buf_idx = um_pci_get_buf (dev , & posted );
145+ buf = & dev -> bufs [buf_idx ];
146+ memcpy (buf , cmd , cmd_size );
147+
148+ if (posted && extra && extra_size > sizeof (buf ) - cmd_size ) {
149+ dev -> extra_ptrs [buf_idx ] = kmemdup (extra , extra_size ,
150+ GFP_ATOMIC );
151+
152+ if (!dev -> extra_ptrs [buf_idx ]) {
153+ um_pci_free_buf (dev , buf );
154+ return - ENOMEM ;
123155 }
156+ extra = dev -> extra_ptrs [buf_idx ];
157+ } else if (extra && extra_size <= sizeof (buf ) - cmd_size ) {
158+ memcpy ((u8 * )buf + cmd_size , extra , extra_size );
159+ cmd_size += extra_size ;
160+ extra_size = 0 ;
161+ extra = NULL ;
162+ cmd = (void * )buf ;
124163 } else {
125164 cmd = (void * )buf ;
126165 }
127166
128167 sg_init_one (& out_sg , cmd , cmd_size );
129168 if (extra )
130169 sg_init_one (& extra_sg , extra , extra_size );
131- if (out )
170+ /* allow stack for small buffers */
171+ if (bounce_out )
172+ sg_init_one (& in_sg , buf -> data , out_size );
173+ else if (out )
132174 sg_init_one (& in_sg , out , out_size );
133175
134176 /* add to internal virtio queue */
135177 ret = virtqueue_add_sgs (dev -> cmd_vq , sgs_list ,
136178 extra ? 2 : 1 ,
137179 out ? 1 : 0 ,
138- posted ? cmd : HANDLE_NO_FREE (cmd ),
139- GFP_ATOMIC );
180+ cmd , GFP_ATOMIC );
140181 if (ret ) {
141- if (posted )
142- kfree (cmd );
143- goto out ;
182+ um_pci_free_buf (dev , buf );
183+ return ret ;
144184 }
145185
146186 if (posted ) {
147187 virtqueue_kick (dev -> cmd_vq );
148- ret = 0 ;
149- goto out ;
188+ return 0 ;
150189 }
151190
152191 /* kick and poll for getting a response on the queue */
153192 set_bit (UM_PCI_STAT_WAITING , & dev -> status );
154193 virtqueue_kick (dev -> cmd_vq );
194+ ret = 0 ;
155195
156196 while (1 ) {
157197 void * completed = virtqueue_get_buf (dev -> cmd_vq , & len );
158198
159- if (completed == HANDLE_NO_FREE ( cmd ) )
199+ if (completed == buf )
160200 break ;
161201
162- if (completed && ! HANDLE_IS_NO_FREE ( completed ) )
163- kfree ( completed );
202+ if (completed )
203+ um_pci_free_buf ( dev , completed );
164204
165205 if (WARN_ONCE (virtqueue_is_broken (dev -> cmd_vq ) ||
166206 ++ delay_count > um_pci_max_delay_us ,
@@ -172,8 +212,11 @@ static int um_pci_send_cmd(struct um_pci_device *dev,
172212 }
173213 clear_bit (UM_PCI_STAT_WAITING , & dev -> status );
174214
175- out :
176- put_cpu_var (um_pci_msg_bufs );
215+ if (bounce_out )
216+ memcpy (out , buf -> data , out_size );
217+
218+ um_pci_free_buf (dev , buf );
219+
177220 return ret ;
178221}
179222
@@ -187,20 +230,13 @@ static unsigned long um_pci_cfgspace_read(void *priv, unsigned int offset,
187230 .size = size ,
188231 .addr = offset ,
189232 };
190- /* buf->data is maximum size - we may only use parts of it */
191- struct um_pci_message_buffer * buf ;
192- u8 * data ;
193- unsigned long ret = ULONG_MAX ;
194- size_t bytes = sizeof (buf -> data );
233+ /* max 8, we might not use it all */
234+ u8 data [8 ];
195235
196236 if (!dev )
197237 return ULONG_MAX ;
198238
199- buf = get_cpu_var (um_pci_msg_bufs );
200- data = buf -> data ;
201-
202- if (buf )
203- memset (data , 0xff , bytes );
239+ memset (data , 0xff , sizeof (data ));
204240
205241 switch (size ) {
206242 case 1 :
@@ -212,34 +248,26 @@ static unsigned long um_pci_cfgspace_read(void *priv, unsigned int offset,
212248 break ;
213249 default :
214250 WARN (1 , "invalid config space read size %d\n" , size );
215- goto out ;
251+ return ULONG_MAX ;
216252 }
217253
218- if (um_pci_send_cmd (dev , & hdr , sizeof (hdr ), NULL , 0 , data , bytes ))
219- goto out ;
254+ if (um_pci_send_cmd (dev , & hdr , sizeof (hdr ), NULL , 0 , data , size ))
255+ return ULONG_MAX ;
220256
221257 switch (size ) {
222258 case 1 :
223- ret = data [0 ];
224- break ;
259+ return data [0 ];
225260 case 2 :
226- ret = le16_to_cpup ((void * )data );
227- break ;
261+ return le16_to_cpup ((void * )data );
228262 case 4 :
229- ret = le32_to_cpup ((void * )data );
230- break ;
263+ return le32_to_cpup ((void * )data );
231264#ifdef CONFIG_64BIT
232265 case 8 :
233- ret = le64_to_cpup ((void * )data );
234- break ;
266+ return le64_to_cpup ((void * )data );
235267#endif
236268 default :
237- break ;
269+ return ULONG_MAX ;
238270 }
239-
240- out :
241- put_cpu_var (um_pci_msg_bufs );
242- return ret ;
243271}
244272
245273static void um_pci_cfgspace_write (void * priv , unsigned int offset , int size ,
@@ -312,13 +340,8 @@ static void um_pci_bar_copy_from(void *priv, void *buffer,
312340static unsigned long um_pci_bar_read (void * priv , unsigned int offset ,
313341 int size )
314342{
315- /* buf->data is maximum size - we may only use parts of it */
316- struct um_pci_message_buffer * buf ;
317- u8 * data ;
318- unsigned long ret = ULONG_MAX ;
319-
320- buf = get_cpu_var (um_pci_msg_bufs );
321- data = buf -> data ;
343+ /* 8 is maximum size - we may only use parts of it */
344+ u8 data [8 ];
322345
323346 switch (size ) {
324347 case 1 :
@@ -330,33 +353,25 @@ static unsigned long um_pci_bar_read(void *priv, unsigned int offset,
330353 break ;
331354 default :
332355 WARN (1 , "invalid config space read size %d\n" , size );
333- goto out ;
356+ return ULONG_MAX ;
334357 }
335358
336359 um_pci_bar_copy_from (priv , data , offset , size );
337360
338361 switch (size ) {
339362 case 1 :
340- ret = data [0 ];
341- break ;
363+ return data [0 ];
342364 case 2 :
343- ret = le16_to_cpup ((void * )data );
344- break ;
365+ return le16_to_cpup ((void * )data );
345366 case 4 :
346- ret = le32_to_cpup ((void * )data );
347- break ;
367+ return le32_to_cpup ((void * )data );
348368#ifdef CONFIG_64BIT
349369 case 8 :
350- ret = le64_to_cpup ((void * )data );
351- break ;
370+ return le64_to_cpup ((void * )data );
352371#endif
353372 default :
354- break ;
373+ return ULONG_MAX ;
355374 }
356-
357- out :
358- put_cpu_var (um_pci_msg_bufs );
359- return ret ;
360375}
361376
362377static void um_pci_bar_copy_to (void * priv , unsigned int offset ,
@@ -523,11 +538,8 @@ static void um_pci_cmd_vq_cb(struct virtqueue *vq)
523538 if (test_bit (UM_PCI_STAT_WAITING , & dev -> status ))
524539 return ;
525540
526- while ((cmd = virtqueue_get_buf (vq , & len ))) {
527- if (WARN_ON (HANDLE_IS_NO_FREE (cmd )))
528- continue ;
529- kfree (cmd );
530- }
541+ while ((cmd = virtqueue_get_buf (vq , & len )))
542+ um_pci_free_buf (dev , cmd );
531543}
532544
533545static void um_pci_irq_vq_cb (struct virtqueue * vq )
@@ -1006,10 +1018,6 @@ static int __init um_pci_init(void)
10061018 "No virtio device ID configured for PCI - no PCI support\n" ))
10071019 return 0 ;
10081020
1009- um_pci_msg_bufs = alloc_percpu (struct um_pci_message_buffer );
1010- if (!um_pci_msg_bufs )
1011- return - ENOMEM ;
1012-
10131021 bridge = pci_alloc_host_bridge (0 );
10141022 if (!bridge ) {
10151023 err = - ENOMEM ;
@@ -1070,7 +1078,6 @@ static int __init um_pci_init(void)
10701078 pci_free_resource_list (& bridge -> windows );
10711079 pci_free_host_bridge (bridge );
10721080 }
1073- free_percpu (um_pci_msg_bufs );
10741081 return err ;
10751082}
10761083module_init (um_pci_init );
@@ -1082,6 +1089,5 @@ static void __exit um_pci_exit(void)
10821089 irq_domain_remove (um_pci_inner_domain );
10831090 pci_free_resource_list (& bridge -> windows );
10841091 pci_free_host_bridge (bridge );
1085- free_percpu (um_pci_msg_bufs );
10861092}
10871093module_exit (um_pci_exit );
0 commit comments