Add text to legend color square in ggplot2

I'm making a diverging bar plot for some Likert data (along these lines). The client has requested a "mean response" for each group, treating Likert responses as consecutive integers ("Strongly disagree" = 1, "Disagree" = 2, etc.); these means are displayed on top of the bars, in the "neutral" area.

For transparency, I want to add the numeric value of each Likert response to the legend. I could add the number to the label (e.g., "Strongly agree (5)"), but I would prefer to put it on top of the color box (e.g., on top of the blue square for "Strongly agree").

Here's the code that produces the diverging bar plot:

library(dplyr)
library(ggplot2)
library(RColorBrewer)
# The data.
df = structure(list(group = structure(c(1L, 2L, 3L, 1L, 2L, 3L, 1L, 2L, 3L, 1L, 2L, 3L, 1L, 2L, 3L),
                                  .Label = c("Group A", "Group B", "Group C"),
                                  class = "factor"),
                response = c(1L, 1L, 1L, 2L, 2L, 2L, 3L, 3L, 3L, 4L, 4L, 4L, 5L, 5L, 5L),
                n = c(4, 8, 25, 8, 25, 29, 29, 35, 28, 25, 22, 12, 34, 10, 6),
                mean.response = c(3.8, 3, 2.5, 3.8, 3, 2.5, 3.8, 3, 2.5, 3.8, 3, 2.5, 3.8, 3, 2.5),
                response.fill = c("#CA0020", "#CA0020", "#CA0020",
                                  "#F4A582", "#F4A582", "#F4A582",
                                  "#F7F7F7", "#F7F7F7", "#F7F7F7",
                                  "#92C5DE", "#92C5DE", "#92C5DE",
                                  "#0571B0", "#0571B0", "#0571B0"),
                n.to.plot = c(4, 8, 25, 8, 25, 29, 14.5, 17.5, 14, 25, 22, 12, 34, 10, 6)),
           class = c("grouped_df", "tbl_df", "tbl", "data.frame"),
           row.names = c(NA, -15L),
           groups = structure(list(group = structure(1:3, .Label = c("Group A", "Group B", "Group C"),
                                                     class = "factor"),
                                   .rows = list(c(1L, 4L, 7L, 10L, 13L),
                                                c(2L, 5L, 8L, 11L, 14L),
                                                c(3L, 6L, 9L, 12L, 15L))),
                              row.names = c(NA, -3L),
                              class = c("tbl_df", "tbl", "data.frame"),
                              .drop = TRUE))
# Groups, responses, and colors.
n.groups = 3
groups = paste("Group", LETTERS[1:n.groups])
likert.responses = c("Strongly disagree", "Disagree", "Neutral", "Agree", "Strongly agree")
pal = brewer.pal(length(likert.responses), "RdBu")
# Make the plot.
ggplot(data = df, aes(x = group, y = n.to.plot, fill = response.fill)) +
  # Start with the "agree" responses.
  geom_bar(data = df %>% filter(response >= 3),
           stat = "identity") +
  # Add the "disagree" responses going the opposite way.
  geom_bar(data = df %>%
             filter(response <= 3) %>%
             mutate(n.to.plot = n.to.plot * -1),
           stat = "identity") +
  # Add text labels with the mean response for each group.
  geom_text(data = df %>%
              dplyr::select(group, mean.response) %>%
              distinct(),
            aes(x = group, y = 0,
                label = format(mean.response, nsmall = 1),
                fill = NA)) +
  # Specify fill colors.
  scale_fill_identity("Response", breaks = pal, labels = likert.responses,
                      guide = "legend") +
  # Adjust axis labels.
  scale_x_discrete("") +
  scale_y_continuous("Number of responses") +
  # Swap x and y axes.
  coord_flip() +
  # Add the prompt text as the title.
  ggtitle("I like program XYZ.")

And here's my desired output:

Taking inspiration from this answer, I tried adding a label aesthetic to the fill legend, but that did nothing:

+ guides(fill = guide_legend(override.aes = list(label = "foo")))

I know I can customizethe shapeof the legend symbols, but the problem is that I want two things: a square with the color, and a black digit superimposed on the square.