'Αυτό το event ενεργοποιείται κάθε φορά που ζωγραφίζεται ένα item       
Private Sub ListBox1_DrawItem(ByVal sender As Object, ByVal e As System.Windows.Forms.DrawItemEventArgs) Handles ListBox1.DrawItem

         'Παίρνουμε το Rectangle του Item που ζωγραφίζεται       
         Dim StringBounds As RectangleF = New RectangleF(e.Bounds.Left, e.Bounds.Top, e.Bounds.Width, e.Bounds.Height)

         'Ελέγχουμε αν είναι Selected ή όχι για να βάλουμε τα αντίστοιχα χρώματα
         If e.Index = sender.SelectedIndex Then
            e.Graphics.FillRectangle(System.Drawing.SystemBrushes.Highlight, e.Bounds)
            e.Graphics.DrawString(sender.Items(e.Index).ToString, sender.Font, System.Drawing.SystemBrushes.HighlightText, StringBounds)

         Else
            e.Graphics.FillRectangle(New SolidBrush(CType(ListBox1.Items(e.Index), Color)), e.Bounds)
            e.Graphics.DrawString(sender.Items(e.Index).ToString, sender.Font, System.Drawing.SystemBrushes.WindowText, StringBounds)

        End If

Εnd Sub

'Στο MouseDown επιλέγουμε το σωστό Selectedindex       
Private Sub ListBox1_MouseDown(ByVal sender As Object, ByVal e As System.Windows.Forms.MouseEventArgs) Handles ListBox1.MouseDown
            Dim lbSender As ListBox = sender
            Dim lastSelectedBounds As Rectangle = lbSender.GetItemRectangle(lbSender.SelectedIndex)  'To προηγούμενο      
            Dim Selectedindex As Integer = lbSender.IndexFromPoint(e.X, e.Y) 'Το τρέχον
            ListBox1.SelectedIndex = Selectedindex 'Αλλάζουμε το SelectedIndex
            ListBox1.Invalidate() 'και ξανακάνουμε redraw το ListBox

End Sub

'Το ίδιο και στο KeyDown
Private Sub ListBox1_KeyDown(ByVal sender As Object, ByVal e As System.Windows.Forms.KeyEventArgs) Handles ListBox1.KeyDown
            Dim selectedindex As Integer = ListBox1.SelectedIndex 'Το παλιό
            Dim lastSelectedBounds As Rectangle = ListBox1.GetItemRectangle(selectedindex)
            Select Case e.KeyCode
                     Case Keys.Up 'πάνω
                              If selectedindex > 0 Then
                                    selectedindex -= 1
                                    ListBox1.SelectedIndex = selectedindex 'Αλλάζουμε το SelectedIndex
                                    ListBox1.Invalidate() 'και ξανακάνουμε redraw το ListBox
                              End If

                     Case Keys.Down 'Κάτω
                             If selectedindex + 1 < ListBox1.Items.Count Then
                                 selectedindex += 1
                                 ListBox1.SelectedIndex = selectedindex
                                 ListBox1.Invalidate()
                           End If

            End Select

End Sub

Για να φορτώσουμε τα χρώματα στο ListBox

Sub LoadColors()
      ListBox1.Items.Add(System.Drawing.Color.Red)
      ListBox1.Items.Add(System.Drawing.Color.Green)
      ListBox1.Items.Add(System.Drawing.Color.Blue)
      ListBox1.Items.Add(System.Drawing.Color.Yellow)
      '.....

End Sub

Καλή επιτυχία !!!!!